Started by user Jenkins Admin Obtained pipelines/pingcap/ticdc/latest/pull_cdc_mysql_integration_light.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_integration_build.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-1j3z2-t8vg2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6acf5c37a2f67eeb85c4c0444b1c16e3bc0c108a" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-1j3z2" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git rev-list --no-walk b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} _=/usr/bin/printenv TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-1j3z2 HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes JENKINS_NODE_COOKIE=77e00bb8-1267-4c99-bf21-8c75808ffc28 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Debug info JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=3 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_ticdc_pull_cdc_mysql_integration_light_358-1j3z2 pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-0d803d454380e64974837c501e5f3c6392293e213080f3de35a070465627e926 + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.23.3' GODEBUG='' GOTELEMETRY='local' GOTELEMETRYDIR='/home/jenkins/.config/go/telemetry' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -ffile-prefix-map=/tmp/go-build2540101894=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-ticdc-pull-cdc-mysql-integration-light-358-1j3z2--f52bp bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 3294 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 723ef215bfeb3c22 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 27 IN A 20.27.177.113 github.com. 27 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Fri Apr 04 07:51:32 UTC 2025 ;; MSG SIZE rcvd: 103 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/ticdc/rev-5d77460-1793b82) 109742080 bytes in 0.46 secs (240492482 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.43.5 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/.git/ .git HEAD is now at 1793b825 Upgrade go-mysql to v1.12.0 POST git-upload-pack (650 bytes) POST git-upload-pack (973 bytes) From https://github.com/pingcap/ticdc = [up to date] master -> origin/master + 1793b825...52a1009d refs/pull/1208/head -> origin/pr/1208/head (forced update) Warning: you are leaving 1 commit behind, not connected to any of your branches: 1793b825 Upgrade go-mysql to v1.12.0 If you want to keep it by creating a new branch, this may be a good time to do so with: git branch 1793b825 HEAD is now at 5d77460e sink: remove ColumnFlagType (#1185) ๐Ÿšง Checkouting to base SHA:5d77460e01078ac672ee4eeeef20afe573266424... HEAD is now at 5d77460e sink: remove ColumnFlagType (#1185) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: 5d77460e01078ac672ee4eeeef20afe573266424 5d77460e sink: remove ColumnFlagType (#1185) 691c5e10 mysql: extract col value meet error then panic immediately (#1203) 23a34e88 codec: add open protocol decoder (#1193) ๐Ÿšง Pre-merge heads of pull requests to base SHA: 5d77460e01078ac672ee4eeeef20afe573266424 ... Updating 5d77460e..52a1009d Fast-forward go.mod | 5 ++--- go.sum | 8 ++------ 2 files changed, 4 insertions(+), 9 deletions(-) ๐Ÿงพ Pre-merged result: 52a1009d7bcd84b283c5443601c9b3160b35bb7c 52a1009d Upgrade go-mysql to v1.12.0 5d77460e sink: remove ColumnFlagType (#1185) 691c5e10 mysql: extract col value meet error then panic immediately (#1203) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache saved successfully (git/pingcap/ticdc/rev-5d77460-52a1009) 109824000 bytes in 1.79 secs (61393390 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 25 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/third_party_download [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] echo computeBranchFromPR component: tidb, prTargetBranch: master, prTitle: Upgrade go-mysql to v1.12.0, trunkBranch: master [Pipeline] echo computeBranchFromPR component: pd, prTargetBranch: master, prTitle: Upgrade go-mysql to v1.12.0, trunkBranch: master [Pipeline] echo computeBranchFromPR component: tikv, prTargetBranch: master, prTitle: Upgrade go-mysql to v1.12.0, trunkBranch: master [Pipeline] echo computeBranchFromPR component: tiflash, prTargetBranch: master, prTitle: Upgrade go-mysql to v1.12.0, trunkBranch: master [Pipeline] retry [Pipeline] { [Pipeline] sh + export TIDB_BRANCH=master + TIDB_BRANCH=master + export PD_BRANCH=master + PD_BRANCH=master + export TIKV_BRANCH=master + TIKV_BRANCH=master + export TIFLASH_BRANCH=master + TIFLASH_BRANCH=master + cd ../ticdc + ./tests/scripts/download-integration-test-binaries.sh master [2025-04-04 15:51:40] Start downloading [2025-04-04 15:51:40] Downloading binaries... >>> Downloading tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/d9fb3a53376157be49b5a7941e6e689a33c81eb0/centos7/tidb-server.tar.gz 2025-04-04 15:52:00 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/d9fb3a53376157be49b5a7941e6e689a33c81eb0/centos7/tidb-server.tar.gz [620275338/620275338] -> "tmp/tidb-server.tar.gz" [1] >>> Downloading pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/9d6bbb8529d9d2acb9cf324a65e94ece465f6346/centos7/pd-server.tar.gz 2025-04-04 15:52:18 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/9d6bbb8529d9d2acb9cf324a65e94ece465f6346/centos7/pd-server.tar.gz [228444265/228444265] -> "tmp/pd-server.tar.gz" [1] >>> Downloading tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/3eb3439fd88f0db9de06ef92e845e80b32c9b49a/centos7/tikv-server.tar.gz 2025-04-04 15:52:43 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/3eb3439fd88f0db9de06ef92e845e80b32c9b49a/centos7/tikv-server.tar.gz [817400808/817400808] -> "tmp/tikv-server.tar.gz" [1] >>> Downloading tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8ecb79fcf30098e6786f4d0440ead2a879a3ad4a/centos7/tiflash.tar.gz 2025-04-04 15:53:08 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8ecb79fcf30098e6786f4d0440ead2a879a3ad4a/centos7/tiflash.tar.gz [553235170/553235170] -> "tmp/tiflash.tar.gz" [1] >>> Downloading minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2025-04-04 15:53:14 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> Downloading etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2025-04-04 15:53:15 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> Downloading sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-a129f096_linux-amd64.tar.gz 2025-04-04 15:53:19 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-a129f096_linux-amd64.tar.gz [59679894/59679894] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> Downloading schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2025-04-04 15:53:30 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] >>> Downloading go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2025-04-04 15:53:35 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> Downloading jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2025-04-04 15:53:35 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] [2025-04-04 15:53:35] Download SUCCESS + ls -alh ./bin total 2.1G drwxr-sr-x 6 jenkins jenkins 4.0K Apr 4 15:53 . drwxr-sr-x 22 jenkins jenkins 4.0K Apr 4 15:53 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M Apr 4 15:53 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M Apr 4 15:53 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1.2M Dec 18 2023 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 3 12:33 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 353K Dec 18 2023 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 3 12:33 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.2M Apr 3 11:49 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 346M Apr 3 12:34 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 41M Apr 3 15:30 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 3 15:29 pd-ctl -rwxr-xr-x 1 jenkins jenkins 40M Apr 3 15:30 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:29 pd-recover -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-region-bench -rwxr-xr-x 1 jenkins jenkins 115M Apr 3 15:29 pd-server -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 15M Apr 3 15:30 pd-ut -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 stores-dump -rwxr-xr-x 1 jenkins jenkins 140M Sep 11 2024 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 241M Apr 3 19:14 tidb-server -rwxr-xr-x 1 jenkins jenkins 443M Apr 3 12:33 tiflash -rwxr-xr-x 1 jenkins jenkins 373M Apr 2 11:56 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.3M Apr 3 15:30 xprog + make check_third_party_binary go: downloading go1.23.7 (linux/amd64) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/tidb-server /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/tikv-server /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/pd-server /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/tiflash /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/etcdctl /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/jq /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/minio /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/third_party_download + mkdir -p bin + mv ../ticdc/bin/bin ../ticdc/bin/etc ../ticdc/bin/etcdctl ../ticdc/bin/go-ycsb ../ticdc/bin/jq ../ticdc/bin/lib ../ticdc/bin/libc++.so.1 ../ticdc/bin/libc++.so.1.0 ../ticdc/bin/libc++abi.so.1 ../ticdc/bin/libc++abi.so.1.0 ../ticdc/bin/libgmssl.so ../ticdc/bin/libgmssl.so.3 ../ticdc/bin/libgmssl.so.3.0 ../ticdc/bin/libtiflash_proxy.so ../ticdc/bin/minio ../ticdc/bin/pd-api-bench ../ticdc/bin/pd-ctl ../ticdc/bin/pd-heartbeat-bench ../ticdc/bin/pd-recover ../ticdc/bin/pd-region-bench ../ticdc/bin/pd-server ../ticdc/bin/pd-tso-bench ../ticdc/bin/pd-ut ../ticdc/bin/regions-dump ../ticdc/bin/share ../ticdc/bin/stores-dump ../ticdc/bin/sync_diff_inspector ../ticdc/bin/tidb-server ../ticdc/bin/tiflash ../ticdc/bin/tikv-server ../ticdc/bin/xprog ./bin/ + ls -alh ./bin total 2.1G drwxr-sr-x 6 jenkins jenkins 4.0K Apr 4 15:53 . drwxr-sr-x 3 jenkins jenkins 4.0K Apr 4 15:53 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M Apr 4 15:53 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M Apr 4 15:53 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1.2M Dec 18 2023 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 3 12:33 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 353K Dec 18 2023 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 3 12:33 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.2M Apr 3 11:49 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 346M Apr 3 12:34 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 41M Apr 3 15:30 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 3 15:29 pd-ctl -rwxr-xr-x 1 jenkins jenkins 40M Apr 3 15:30 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:29 pd-recover -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-region-bench -rwxr-xr-x 1 jenkins jenkins 115M Apr 3 15:29 pd-server -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 15M Apr 3 15:30 pd-ut -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 stores-dump -rwxr-xr-x 1 jenkins jenkins 140M Sep 11 2024 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 241M Apr 3 19:14 tidb-server -rwxr-xr-x 1 jenkins jenkins 443M Apr 3 12:33 tiflash -rwxr-xr-x 1 jenkins jenkins 373M Apr 2 11:56 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.3M Apr 3 15:30 xprog + ./bin/tidb-server -V Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 + ./bin/tikv-server -V TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x 2 jenkins jenkins 4.0K Apr 4 15:53 . drwxr-sr-x 22 jenkins jenkins 4.0K Apr 4 15:53 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/ticdc/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/ticdc/pkg/version.BuildTS=2025-04-04 07:53:41" -X "github.com/pingcap/ticdc/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/ticdc/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/ticdc/pkg/version.GoVersion=go version go1.23.7 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:53:41"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading google.golang.org/grpc v1.65.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/spf13/cobra v1.8.1 go: downloading github.com/pingcap/tiflow v0.0.0-20250318113613-e23fa78655fa go: downloading github.com/fatih/color v1.18.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading golang.org/x/time v0.7.0 go: downloading golang.org/x/net v0.33.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/r3labs/diff v1.1.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/cockroachdb/pebble v1.1.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/json-iterator/go v1.1.12 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/imdario/mergo v0.3.16 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.24.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 go: downloading golang.org/x/sys v0.28.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 go: downloading github.com/cockroachdb/errors v1.11.3 go: downloading github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/go-mysql-org/go-mysql v1.12.0 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.22.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading golang.org/x/text v0.21.0 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading go.opencensus.io v0.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading gorm.io/gorm v1.25.11 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading filippo.io/edwards25519 v1.1.0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/golang-jwt/jwt/v4 v4.5.1 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/go-playground/locales v0.14.1 go: downloading sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading k8s.io/api v0.29.11 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading cloud.google.com/go/kms v1.15.8 go: downloading github.com/golang/glog v1.2.1 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 go: downloading k8s.io/apimachinery v0.29.11 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/ticdc/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/ticdc/pkg/version.BuildTS=2025-04-04 07:55:46" -X "github.com/pingcap/ticdc/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/ticdc/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/ticdc/pkg/version.GoVersion=go version go1.23.7 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:55:46"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=1 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/ticdc/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/ticdc/pkg/version.BuildTS=2025-04-04 07:55:46" -X "github.com/pingcap/ticdc/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/ticdc/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/ticdc/pkg/version.GoVersion=go version go1.23.7 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:55:46"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer go: downloading github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 go: downloading github.com/DataDog/zstd v1.5.5 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/ticdc/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/ticdc/pkg/version.BuildTS=2025-04-04 07:55:46" -X "github.com/pingcap/ticdc/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/ticdc/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/ticdc/pkg/version.GoVersion=go version go1.23.7 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:55:46"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/ticdc/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/ticdc/pkg/version.BuildTS=2025-04-04 07:55:46" -X "github.com/pingcap/ticdc/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/ticdc/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/ticdc/pkg/version.GoVersion=go version go1.23.7 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:55:46"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|ticdc/tests|integration|testing_utils|pb|pbmock|ticdc/bin'); do echo ${p#"github.com/pingcap/ticdc/"}|grep -v "github.com/pingcap/ticdc"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/ticdc/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/ticdc/pkg/version.BuildTS=2025-04-04 07:55:46" -X "github.com/pingcap/ticdc/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/ticdc/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/ticdc/pkg/version.GoVersion=go version go1.23.7 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:55:46"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/ticdc/... \ -o bin/cdc.test github.com/pingcap/ticdc/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|ticdc/tests|integration|testing_utils|pb|pbmock|ticdc/bin'); do echo ${p#"github.com/pingcap/ticdc/"}|grep -v "github.com/pingcap/ticdc"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); echo "Failed to build cdc.test"; exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/ticdc/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/ticdc/pkg/version.BuildTS=2025-04-04 07:55:46" -X "github.com/pingcap/ticdc/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/ticdc/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/ticdc/pkg/version.GoVersion=go version go1.23.7 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-55-g52a1009d" -X "github.com/pingcap/tiflow/pkg/version.GitHash=52a1009d7bcd84b283c5443601c9b3160b35bb7c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:55:46"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|ticdc/tests|integration|testing_utils|pb|pbmock|ticdc/bin'); do echo ${p#"github.com/pingcap/ticdc/"}|grep -v "github.com/pingcap/ticdc"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|ticdc/tests|integration|testing_utils|pb|pbmock|ticdc/bin'); do echo ${p#"github.com/pingcap/ticdc/"}|grep -v "github.com/pingcap/ticdc"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.3G drwxr-sr-x 2 jenkins jenkins 4.0K Apr 4 16:02 . drwxr-sr-x 22 jenkins jenkins 4.0K Apr 4 15:53 .. -rwxr-xr-x 1 jenkins jenkins 242M Apr 4 16:02 cdc -rwxr-xr-x 1 jenkins jenkins 383M Apr 4 16:02 cdc.test -rwxr-xr-x 1 jenkins jenkins 269M Apr 4 15:57 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 200M Apr 4 15:57 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 200M Apr 4 15:55 cdc_storage_consumer -rwxr-xr-x 1 jenkins jenkins 13M Apr 4 15:58 oauth2-server + ./bin/cdc version Release Version: v9.0.0-beta.1-55-g52a1009d Git Commit Hash: 52a1009d7bcd84b283c5443601c9b3160b35bb7c Git Branch: HEAD UTC Build Time: 2025-04-04 07:55:46 Go Version: go version go1.23.7 linux/amd64 Failpoint Build: false [Pipeline] } Cache saved successfully (binary/pingcap/ticdc/cdc-mysql-integration/rev-5d77460-52a1009) 1367142400 bytes in 16.50 secs (82839854 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-region-bench ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.4G drwxr-sr-x 6 jenkins jenkins 4.0K Apr 4 16:02 . drwxr-sr-x 22 jenkins jenkins 4.0K Apr 4 15:53 .. drwxr-sr-x 2 jenkins jenkins 4.0K Apr 4 16:02 bin -rwxr-xr-x 1 jenkins jenkins 242M Apr 4 16:02 cdc -rwxr-xr-x 1 jenkins jenkins 383M Apr 4 16:02 cdc.test -rwxr-xr-x 1 jenkins jenkins 269M Apr 4 15:57 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 200M Apr 4 15:57 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 200M Apr 4 15:55 cdc_storage_consumer drwxr-sr-x 4 jenkins jenkins 4.0K Apr 4 16:02 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 4 16:02 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M Apr 4 16:02 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M Apr 4 16:02 jq drwxr-sr-x 3 jenkins jenkins 4.0K Apr 4 16:02 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 4 16:02 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1.2M Apr 4 16:02 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 4 16:02 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 353K Apr 4 16:02 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 4 16:02 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 4 16:02 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.2M Apr 4 16:02 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 346M Apr 4 16:02 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Apr 4 16:02 minio -rwxr-xr-x 1 jenkins jenkins 13M Apr 4 15:58 oauth2-server -rwxr-xr-x 1 jenkins jenkins 41M Apr 4 16:02 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 4 16:02 pd-ctl -rwxr-xr-x 1 jenkins jenkins 40M Apr 4 16:02 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 35M Apr 4 16:02 pd-recover -rwxr-xr-x 1 jenkins jenkins 37M Apr 4 16:02 pd-region-bench -rwxr-xr-x 1 jenkins jenkins 115M Apr 4 16:02 pd-server -rwxr-xr-x 1 jenkins jenkins 37M Apr 4 16:02 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 15M Apr 4 16:02 pd-ut -rwxr-xr-x 1 jenkins jenkins 35M Apr 4 16:02 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K Apr 4 16:02 share -rwxr-xr-x 1 jenkins jenkins 35M Apr 4 16:02 stores-dump -rwxr-xr-x 1 jenkins jenkins 140M Apr 4 16:02 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 241M Apr 4 16:02 tidb-server -rwxr-xr-x 1 jenkins jenkins 443M Apr 4 16:02 tiflash -rwxr-xr-x 1 jenkins jenkins 373M Apr 4 16:02 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.3M Apr 4 16:02 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 50.41 secs (78877855 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420 Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4 Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-wvb9t-ltbrk --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "441eff45cd3a640bc2d4ecc3b4628120d94c6124" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-wvb9t" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light [Pipeline] { [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420 is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-4l8h3-67bfz --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "418c60ad10a232d5deabf459e4b86747f8815cb6" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-4l8h3" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420 in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4 is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-63s8p-p4kmb --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "df70d3ef803ffbe7c500fefa7f9b5fbb1f2dc506" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-63s8p" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4 in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light [Pipeline] { [Pipeline] checkout No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@114930f3; decorates RemoteLauncher[hudson.remoting.Channel@1ab0fb81:JNLP4-connect connection from 10.233.106.222/10.233.106.222:38072] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@ce6f074; decorates RemoteLauncher[hudson.remoting.Channel@1a8ad9e6:JNLP4-connect connection from 10.233.76.229/10.233.76.229:41898] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3b8d4f8e; decorates RemoteLauncher[hudson.remoting.Channel@6bf48e36:JNLP4-connect connection from 10.233.108.208/10.233.108.208:40548] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] node Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-7m7s5-sgwfq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "be0db13525a8457aae372cdeb5f2db94178b031a" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-7m7s5" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq [Pipeline] checkout The recommended git tool is: git Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-0xql0-pjrvb --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "df6d7321034648baeb3b9cc9beb1f70c8027b380" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-0xql0" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light [Pipeline] { [Pipeline] checkout The recommended git tool is: git Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-w6z00-tfr53 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3aac7ef7e37dacb41666c2dd969fd74ec4f83197" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-w6z00" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) [Pipeline] node > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" [Pipeline] withEnv [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8 Avoid second fetch [Pipeline] container Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) [Pipeline] { [Pipeline] stage [Pipeline] { (Test) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2d455e; decorates RemoteLauncher[hudson.remoting.Channel@14c6e4a2:JNLP4-connect connection from 10.233.84.72/10.233.84.72:45040] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] timeout Timeout set to expire in 40 min > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] cache Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@16a9395e; decorates RemoteLauncher[hudson.remoting.Channel@380e2953:JNLP4-connect connection from 10.233.88.164/10.233.88.164:51098] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6e154c0b; decorates RemoteLauncher[hudson.remoting.Channel@41caa212:JNLP4-connect connection from 10.233.93.8/10.233.93.8:52662] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-3mwl6-1tbz2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ee350dbba6671650a1ae4fc8ad359b5f5f1d6ff4" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-3mwl6" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-c2312-bd0mz --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "41b3e14100e4a670dc9ec6e8d86fa72d74dffe68" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-c2312" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8 is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-90s3w-2nklg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a386abe81120f9f308297b4bd3065e3e3a4ea450" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-90s3w" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8 in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light Obtained pipelines/pingcap/ticdc/latest/pod-pull_cdc_mysql_integration_light.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-bjkgt-32l89 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3bab7477113115eacd7862356f397f9148302d39" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-bjkgt" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 7.36 secs (540144674 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { + ./tests/integration_tests/run_light_it_in_ci.sh mysql G01 [Pipeline] withEnv Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-qqkbz-91gbj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1747b96149662aaa52d9e5edf5ecc8dac8e9ebaa" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-qqkbz" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] { [Pipeline] container Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G01 Group Number (parsed): 01 Run cases: common_1 foreign_key changefeed_pause_resume fail_over_ddl_B GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-wvb9t HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=acca0fb0-39fa-4733-9e17-c02d56785af5 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G01 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj pingcap_ticdc_pull_cdc_mysql_integration_light_358-wvb9t PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/common_1/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] { The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { The recommended git tool is: git [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6ccb2dfc; decorates RemoteLauncher[hudson.remoting.Channel@576d03ec:JNLP4-connect connection from 10.233.127.218/10.233.127.218:51816] will be ignored (a typical symptom is the Git executable not being run inside a designated container) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3a286091; decorates RemoteLauncher[hudson.remoting.Channel@53425903:JNLP4-connect connection from 10.233.97.221/10.233.97.221:48826] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4ae90317; decorates RemoteLauncher[hudson.remoting.Channel@23885940:JNLP4-connect connection from 10.233.106.88/10.233.106.88:58620] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@109c0721; decorates RemoteLauncher[hudson.remoting.Channel@34127e70:JNLP4-connect connection from 10.233.127.118/10.233.127.118:35024] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5c645823; decorates RemoteLauncher[hudson.remoting.Channel@7c327ec1:JNLP4-connect connection from 10.233.71.123/10.233.71.123:47388] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-n1qgb-mnf4j --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a4e92c026fb99c391e874e2f88249d79dd8038e4" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-n1qgb" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9 is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-sddr0-gx5dr --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3a1addb8ff95e47ba1a7c205f428a6668b901f93" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-sddr0" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-6mcfg-dpp0g --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6301b6c97c4b40e04a167a9ce9802d7f899a66a8" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-6mcfg" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf" - name: "JENKINS_NAME" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9 in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-hbpq7-m6f6v Agent pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs is provisioned from template pingcap_ticdc_pull_cdc_mysql_integration_light_358-fkq8d-v788d --- apiVersion: "v1" kind: "Pod" metadata: annotations: --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" jenkins/label-digest: "ed72329305519b5da099791687dd40328d8ab8b8" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-fkq8d" runUrl: "job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/" labels: jenkins/jenkins-jenkins-agent: "true" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: jenkins/label-digest: "6a71c2cc2af84b8edbc74d397a540d8149537520" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" jenkins/label: "pingcap_ticdc_pull_cdc_mysql_integration_light_358-hbpq7" name: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l" operator: "In" values: - "amd64" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: limits: memory: "16Gi" cpu: "4" tty: true volumeMounts: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" memory: "16Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" name: "net-tool" resources: limits: value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" memory: "128Mi" cpu: "100m" tty: true volumeMounts: - name: "JENKINS_AGENT_NAME" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs" - name: "JENKINS_NAME" - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs" - name: "JENKINS_AGENT_WORKDIR" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l" - name: "JENKINS_NAME" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" value: "pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light Running on pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 7.66 secs (518854690 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489365835251779721 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5224d0000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:1156, start at 2025-04-04 16:04:33.731488148 +0800 CST m=+3.572294643 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:06:33.740 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:04:33.739 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:54:33.739 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489365835251779721 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5224d0000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:1156, start at 2025-04-04 16:04:33.731488148 +0800 CST m=+3.572294643 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:06:33.740 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:04:33.739 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:54:33.739 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489365831280491719 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5226600003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:1240, start at 2025-04-04 16:04:33.819981659 +0800 CST m=+3.487714065 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:06:33.831 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:04:33.840 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:54:33.840 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.2593.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457114616368726017 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114616368726017 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:04:38 CST 2025] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.26282630.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 6.55 secs (606972888 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:04:41 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d63bd5b5-0814-4551-a15e-a12ea4f6834f {"id":"d63bd5b5-0814-4551-a15e-a12ea4f6834f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753878} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd4635fe3 d63bd5b5-0814-4551-a15e-a12ea4f6834f /tidb/cdc/default/__cdc_meta__/owner/223195ffd4635fe3 d63bd5b5-0814-4551-a15e-a12ea4f6834f' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d63bd5b5-0814-4551-a15e-a12ea4f6834f {"id":"d63bd5b5-0814-4551-a15e-a12ea4f6834f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753878} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd4635fe3 d63bd5b5-0814-4551-a15e-a12ea4f6834f /tidb/cdc/default/__cdc_meta__/owner/223195ffd4635fe3 d63bd5b5-0814-4551-a15e-a12ea4f6834f' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d63bd5b5-0814-4551-a15e-a12ea4f6834f {"id":"d63bd5b5-0814-4551-a15e-a12ea4f6834f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753878} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd4635fe3 d63bd5b5-0814-4551-a15e-a12ea4f6834f /tidb/cdc/default/__cdc_meta__/owner/223195ffd4635fe3 d63bd5b5-0814-4551-a15e-a12ea4f6834f' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 1781970043635112785610473535750378555067 Info: {"upstream_id":7489365835251779721,"id":"1781970043635112785610473535750378555067","namespace":"default","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-04T16:04:41.64343029+08:00","start_ts":457114616368726017,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114616368726017,"checkpoint_ts":457114616368726017,"checkpoint_time":"2025-04-04 16:04:37.139","gid":{"low":17819700436351127856,"high":10473535750378555067}} table common_1.v1 not exists for 1-th check, retry later table common_1.v1 not exists for 2-th check, retry later table common_1.v1 exists table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later table common_1.finish_mark not exists for 2-th check, retry later Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 8.36 secs (475898241 bytes/sec) [Pipeline] { [Pipeline] cache table common_1.finish_mark exists check diff failed 1-th time, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/common_1/stdout.log no DATA RACE found [Fri Apr 4 16:04:53 CST 2025] <<<<<< run test case common_1 success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 6.74 secs (589575374 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 7.23 secs (549761304 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + ./tests/integration_tests/run_light_it_in_ci.sh mysql G00 [Pipeline] sh Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G00 Group Number (parsed): 00 Run cases: charset_gbk changefeed_finish sql_mode changefeed_reconstruct fail_over_ddl_A GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-4l8h3 HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=3501b62e-4014-4e1a-bab2-95443096f38f GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G00 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420 pingcap_ticdc_pull_cdc_mysql_integration_light_358-4l8h3 PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/charset_gbk/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/charset_gbk ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/charset_gbk + source /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/charset_gbk/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/charset_gbk + CDC_BINARY=cdc.test + SINK_TYPE=mysql + trap stop_tidb_cluster EXIT + run mysql + rm -rf /tmp/tidb_cdc_test/charset_gbk + mkdir -p /tmp/tidb_cdc_test/charset_gbk + start_tidb_cluster --workdir /tmp/tidb_cdc_test/charset_gbk The 1 times to try to start tidb cluster... + ./tests/integration_tests/run_light_it_in_ci.sh mysql G02 [Pipeline] sh Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G02 Group Number (parsed): 02 Run cases: new_ci_collation safe_mode savepoint fail_over_ddl_C GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-63s8p HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=5c26192b-b803-4f7b-ba0f-a2b7730c86b9 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G02 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4 pingcap_ticdc_pull_cdc_mysql_integration_light_358-63s8p PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + ./tests/integration_tests/run_light_it_in_ci.sh mysql G03 [Pipeline] sh Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G03 Group Number (parsed): 03 Run cases: capture_suicide_while_balance_table kv_client_stream_reconnect fail_over_ddl_D GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-7m7s5 HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=94aa07fc-f56d-4183-9430-2fbf3d856179 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G03 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz pingcap_ticdc_pull_cdc_mysql_integration_light_358-7m7s5 PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/capture_suicide_while_balance_table/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/capture_suicide_while_balance_table ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/capture_suicide_while_balance_table + source /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/capture_suicide_while_balance_table/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/capture_suicide_while_balance_table + CDC_BINARY=cdc.test + SINK_TYPE=mysql + trap stop_tidb_cluster EXIT + run mysql + '[' mysql '!=' mysql ']' + rm -rf /tmp/tidb_cdc_test/capture_suicide_while_balance_table + mkdir -p /tmp/tidb_cdc_test/capture_suicide_while_balance_table + start_tidb_cluster --workdir /tmp/tidb_cdc_test/capture_suicide_while_balance_table The 1 times to try to start tidb cluster... + ./tests/integration_tests/run_light_it_in_ci.sh mysql G04 Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G04 Group Number (parsed): 04 Run cases: ci_collation_compatibility multi_capture resourcecontrol fail_over_ddl_E GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-0xql0 HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=37a445e6-8558-4379-b0b4-4f2cefa7f8ff GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G04 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_ticdc_pull_cdc_mysql_integration_light_358-0xql0 pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/ci_collation_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + ./tests/integration_tests/run_light_it_in_ci.sh mysql G05 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G05 Group Number (parsed): 05 Run cases: vector simple partition_table fail_over_ddl_F GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-w6z00 HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=66405226-eef5-4844-87b1-76c6dbb0e20b GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G05 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_ticdc_pull_cdc_mysql_integration_light_358-w6z00 pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/vector/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@775c10d6; decorates RemoteLauncher[hudson.remoting.Channel@3df20e03:JNLP4-connect connection from 10.233.78.195/10.233.78.195:51082] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@32c2dea7; decorates RemoteLauncher[hudson.remoting.Channel@7e77a452:JNLP4-connect connection from 10.233.108.26/10.233.108.26:44802] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@eee7814; decorates RemoteLauncher[hudson.remoting.Channel@6340ac52:JNLP4-connect connection from 10.233.88.171/10.233.88.171:34500] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@50371481; decorates RemoteLauncher[hudson.remoting.Channel@754ee465:JNLP4-connect connection from 10.233.84.5/10.233.84.5:60508] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@c00f00b; decorates RemoteLauncher[hudson.remoting.Channel@4ac5afee:JNLP4-connect connection from 10.233.70.39/10.233.70.39:44864] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/vector Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) start tidb cluster in /tmp/tidb_cdc_test/ci_collation_compatibility Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366010247407754 TiDB Cluster ID. tikv_gc_leader_uuid 657ff54a700000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:3811, start at 2025-04-04 16:05:14.835612691 +0800 CST m=+3.376664945 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:14.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:14.816 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:14.816 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366010247407754 TiDB Cluster ID. tikv_gc_leader_uuid 657ff54a700000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:3811, start at 2025-04-04 16:05:14.835612691 +0800 CST m=+3.376664945 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:14.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:14.816 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:14.816 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489365998671721106 TiDB Cluster ID. tikv_gc_leader_uuid 657ff54a8900001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:3887, start at 2025-04-04 16:05:14.916957112 +0800 CST m=+3.359127124 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:14.925 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:14.916 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:14.916 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.5088.out cli tso query --pd=http://127.0.0.1:2379 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114627188719617 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114627188719617 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:05:19 CST 2025] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.51285130.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 8.06 secs (493360320 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366031413552643 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5503580002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:1228, start at 2025-04-04 16:05:20.728758137 +0800 CST m=+3.596928687 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:20.737 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:20.734 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:20.734 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366031413552643 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5503580002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:1228, start at 2025-04-04 16:05:20.728758137 +0800 CST m=+3.596928687 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:20.737 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:20.734 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:20.734 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366035477481707 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5506a00003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:1300, start at 2025-04-04 16:05:20.939437771 +0800 CST m=+3.682332693 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:20.949 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:20.936 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:20.936 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366028382142948 TiDB Cluster ID. tikv_gc_leader_uuid 657ff55046c0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:1300, start at 2025-04-04 16:05:20.836408283 +0800 CST m=+3.191926096 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:20.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:20.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:20.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366028382142948 TiDB Cluster ID. tikv_gc_leader_uuid 657ff55046c0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:1300, start at 2025-04-04 16:05:20.836408283 +0800 CST m=+3.191926096 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:20.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:20.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:20.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366034240848167 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5505400002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:1372, start at 2025-04-04 16:05:20.849366651 +0800 CST m=+3.081141743 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:20.857 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:20.848 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:20.848 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366036167173390 TiDB Cluster ID. tikv_gc_leader_uuid 657ff550d000001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:1227, start at 2025-04-04 16:05:21.345459215 +0800 CST m=+3.342143882 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:21.354 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:21.344 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:21.344 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366036167173390 TiDB Cluster ID. tikv_gc_leader_uuid 657ff550d000001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:1227, start at 2025-04-04 16:05:21.345459215 +0800 CST m=+3.342143882 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:21.354 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:21.344 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:21.344 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366040368069036 TiDB Cluster ID. tikv_gc_leader_uuid 657ff551020000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:1303, start at 2025-04-04 16:05:21.566768908 +0800 CST m=+3.400483005 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:21.574 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:21.568 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:21.568 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366034234721657 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5507740010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:1281, start at 2025-04-04 16:05:21.020472279 +0800 CST m=+2.996153783 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:21.088 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:21.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:21.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366034234721657 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5507740010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:1281, start at 2025-04-04 16:05:21.020472279 +0800 CST m=+2.996153783 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:21.088 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:21.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:21.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366029623687919 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5510d80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:1362, start at 2025-04-04 16:05:21.602788091 +0800 CST m=+3.395250432 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:21.610 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:21.590 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:21.590 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ci_collation_compatibility/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:05:22 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1bfeee08-54c9-4a96-8021-83ef4415c628 {"id":"1bfeee08-54c9-4a96-8021-83ef4415c628","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753920} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd50883e6 1bfeee08-54c9-4a96-8021-83ef4415c628 /tidb/cdc/default/__cdc_meta__/owner/223195ffd50883e6 1bfeee08-54c9-4a96-8021-83ef4415c628' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1bfeee08-54c9-4a96-8021-83ef4415c628 {"id":"1bfeee08-54c9-4a96-8021-83ef4415c628","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753920} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd50883e6 1bfeee08-54c9-4a96-8021-83ef4415c628 /tidb/cdc/default/__cdc_meta__/owner/223195ffd50883e6 1bfeee08-54c9-4a96-8021-83ef4415c628' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1bfeee08-54c9-4a96-8021-83ef4415c628 {"id":"1bfeee08-54c9-4a96-8021-83ef4415c628","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753920} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd50883e6 1bfeee08-54c9-4a96-8021-83ef4415c628 /tidb/cdc/default/__cdc_meta__/owner/223195ffd50883e6 1bfeee08-54c9-4a96-8021-83ef4415c628' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.5188.out cli changefeed create --start-ts=457114627188719617 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 578536532152385832513519917119623580593 Info: {"upstream_id":7489366010247407754,"id":"578536532152385832513519917119623580593","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:05:23.152549582+08:00","start_ts":457114627188719617,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114627188719617,"checkpoint_ts":457114627188719617,"checkpoint_time":"2025-04-04 16:05:18.414","gid":{"low":5785365321523858325,"high":13519917119623580593}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366045077596167 TiDB Cluster ID. tikv_gc_leader_uuid 657ff551b0c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:1296, start at 2025-04-04 16:05:22.332781542 +0800 CST m=+3.147036043 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:22.340 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:22.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:22.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366045077596167 TiDB Cluster ID. tikv_gc_leader_uuid 657ff551b0c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:1296, start at 2025-04-04 16:05:22.332781542 +0800 CST m=+3.147036043 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:22.340 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:22.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:22.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366034464568993 TiDB Cluster ID. tikv_gc_leader_uuid 657ff551e34000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:1372, start at 2025-04-04 16:05:22.471636643 +0800 CST m=+3.123070389 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:22.540 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:22.545 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:22.545 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/vector/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/vector/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/vector/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/vector/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/vector/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.cli.2585.out cli tso query --pd=http://127.0.0.1:2379 + cd /tmp/tidb_cdc_test/charset_gbk ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.cli.2575.out cli tso query --pd=http://127.0.0.1:2379 + set +x + cd /tmp/tidb_cdc_test/capture_suicide_while_balance_table + pd_addr=http://127.0.0.1:2379 + run_cdc_server --workdir /tmp/tidb_cdc_test/capture_suicide_while_balance_table --binary cdc.test --pd http://127.0.0.1:2379 --logsuffix 1 --addr 127.0.0.1:8300 [Fri Apr 4 16:05:24 CST 2025] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.25662568.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ci_collation_compatibility.cli.2579.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457114628688183297 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114628688183297 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114628688183297 + run_cdc_server --workdir /tmp/tidb_cdc_test/charset_gbk --binary cdc.test [Fri Apr 4 16:05:25 CST 2025] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.26162618.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.cli.2583.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457114628795924482 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114628795924482 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:05:25 CST 2025] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.26192621.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114628912578561 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114628912578561 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:05:26 CST 2025] <<<<<< START cdc server in ci_collation_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ci_collation_compatibility.26052607.out server --log-file /tmp/tidb_cdc_test/ci_collation_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ci_collation_compatibility/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114629214830594 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114629214830594 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:05:27 CST 2025] <<<<<< START cdc server in vector case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.26112613.out server --log-file /tmp/tidb_cdc_test/vector/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/vector/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:05:27 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36aa8988-7dd9-4897-8139-1ef5005ee015 {"id":"36aa8988-7dd9-4897-8139-1ef5005ee015","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753924} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/owner/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36aa8988-7dd9-4897-8139-1ef5005ee015 {"id":"36aa8988-7dd9-4897-8139-1ef5005ee015","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753924} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/owner/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36aa8988-7dd9-4897-8139-1ef5005ee015 {"id":"36aa8988-7dd9-4897-8139-1ef5005ee015","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753924} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/owner/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015' + grep -q 'etcd info' + break + set +x + export 'GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/sink/mysql/MySQLSinkHangLongTime=1*return(true)' + GO_FAILPOINTS='github.com/pingcap/ticdc/downstreamadapter/sink/mysql/MySQLSinkHangLongTime=1*return(true)' + run_cdc_server --workdir /tmp/tidb_cdc_test/capture_suicide_while_balance_table --binary cdc.test --pd http://127.0.0.1:2379 --logsuffix 2 --addr 127.0.0.1:8301 [Fri Apr 4 16:05:27 CST 2025] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/downstreamadapter/sink/mysql/MySQLSinkHangLongTime=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.26262628.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:05:28 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20549518-cea0-4804-a6a0-d6cb4c6f7722 {"id":"20549518-cea0-4804-a6a0-d6cb4c6f7722","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753925} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52292e4 20549518-cea0-4804-a6a0-d6cb4c6f7722 /tidb/cdc/default/__cdc_meta__/owner/223195ffd52292e4 20549518-cea0-4804-a6a0-d6cb4c6f7722' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20549518-cea0-4804-a6a0-d6cb4c6f7722 {"id":"20549518-cea0-4804-a6a0-d6cb4c6f7722","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753925} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52292e4 20549518-cea0-4804-a6a0-d6cb4c6f7722 /tidb/cdc/default/__cdc_meta__/owner/223195ffd52292e4 20549518-cea0-4804-a6a0-d6cb4c6f7722' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20549518-cea0-4804-a6a0-d6cb4c6f7722 {"id":"20549518-cea0-4804-a6a0-d6cb4c6f7722","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753925} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52292e4 20549518-cea0-4804-a6a0-d6cb4c6f7722 /tidb/cdc/default/__cdc_meta__/owner/223195ffd52292e4 20549518-cea0-4804-a6a0-d6cb4c6f7722' + grep -q 'etcd info' + break + set +x + TOPIC_NAME=ticdc-charset-gbk-14485 + case $SINK_TYPE in + SINK_URI=mysql://normal:123456@127.0.0.1:3306/ + SINK_URI=mysql://normal:123456@127.0.0.1:3306/ + echo '==================== create gbk changefeed ====================' ==================== create gbk changefeed ==================== + which cdc /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc + cdc cli changefeed create --start-ts=457114628688183297 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 37672576369612112719391754537437449623 Info: {"upstream_id":7489366031413552643,"id":"37672576369612112719391754537437449623","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:05:28.650392844+08:00","start_ts":457114628688183297,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114628688183297,"checkpoint_ts":457114628688183297,"checkpoint_time":"2025-04-04 16:05:24.134","gid":{"low":3767257636961211271,"high":9391754537437449623}} + case $SINK_TYPE in + run_sql_file /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/charset_gbk/data/test.sql 127.0.0.1 4000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:05:28 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/401a9c53-f9eb-440d-8325-2276ca15bbec {"id":"401a9c53-f9eb-440d-8325-2276ca15bbec","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753926} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52044ec 401a9c53-f9eb-440d-8325-2276ca15bbec /tidb/cdc/default/__cdc_meta__/owner/223195ffd52044ec 401a9c53-f9eb-440d-8325-2276ca15bbec' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/401a9c53-f9eb-440d-8325-2276ca15bbec {"id":"401a9c53-f9eb-440d-8325-2276ca15bbec","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753926} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52044ec 401a9c53-f9eb-440d-8325-2276ca15bbec /tidb/cdc/default/__cdc_meta__/owner/223195ffd52044ec 401a9c53-f9eb-440d-8325-2276ca15bbec' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/401a9c53-f9eb-440d-8325-2276ca15bbec {"id":"401a9c53-f9eb-440d-8325-2276ca15bbec","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753926} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52044ec 401a9c53-f9eb-440d-8325-2276ca15bbec /tidb/cdc/default/__cdc_meta__/owner/223195ffd52044ec 401a9c53-f9eb-440d-8325-2276ca15bbec' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 101105350197062873465520239591394503829 Info: {"upstream_id":7489366028382142948,"id":"101105350197062873465520239591394503829","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-04T16:05:29.01592516+08:00","start_ts":457114628795924482,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114628795924482,"checkpoint_ts":457114628795924482,"checkpoint_time":"2025-04-04 16:05:24.545","gid":{"low":10110535019706287346,"high":5520239591394503829}} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:05:29 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ae5c830-af13-4081-b6df-25259a00653c {"id":"0ae5c830-af13-4081-b6df-25259a00653c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753926} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52232e5 0ae5c830-af13-4081-b6df-25259a00653c /tidb/cdc/default/__cdc_meta__/owner/223195ffd52232e5 0ae5c830-af13-4081-b6df-25259a00653c' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ae5c830-af13-4081-b6df-25259a00653c {"id":"0ae5c830-af13-4081-b6df-25259a00653c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753926} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52232e5 0ae5c830-af13-4081-b6df-25259a00653c /tidb/cdc/default/__cdc_meta__/owner/223195ffd52232e5 0ae5c830-af13-4081-b6df-25259a00653c' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ae5c830-af13-4081-b6df-25259a00653c {"id":"0ae5c830-af13-4081-b6df-25259a00653c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753926} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52232e5 0ae5c830-af13-4081-b6df-25259a00653c /tidb/cdc/default/__cdc_meta__/owner/223195ffd52232e5 0ae5c830-af13-4081-b6df-25259a00653c' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ci_collation_compatibility.cli.2662.out cli changefeed create --start-ts=457114628912578561 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?safe-mode=true' Create changefeed successfully! ID: 304893642961445548510581410554407341225 Info: {"upstream_id":7489366034234721657,"id":"304893642961445548510581410554407341225","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-04T16:05:29.721718987+08:00","start_ts":457114628912578561,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114628912578561,"checkpoint_ts":457114628912578561,"checkpoint_time":"2025-04-04 16:05:24.990","gid":{"low":3048936429614455485,"high":10581410554407341225}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 7.20 secs (552171064 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:05:30 GMT < Content-Length: 1184 < Content-Type: text/plain; charset=utf-8 < { [1184 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/29ca147e-169a-4e8e-b00f-3ce52224a98a {"id":"29ca147e-169a-4e8e-b00f-3ce52224a98a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753928} /tidb/cdc/default/__cdc_meta__/capture/36aa8988-7dd9-4897-8139-1ef5005ee015 {"id":"36aa8988-7dd9-4897-8139-1ef5005ee015","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753924} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5203138 29ca147e-169a-4e8e-b00f-3ce52224a98a /tidb/cdc/default/__cdc_meta__/owner/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5203138 29ca147e-169a-4e8e-b00f-3ce52224a98a' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/29ca147e-169a-4e8e-b00f-3ce52224a98a {"id":"29ca147e-169a-4e8e-b00f-3ce52224a98a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753928} /tidb/cdc/default/__cdc_meta__/capture/36aa8988-7dd9-4897-8139-1ef5005ee015 {"id":"36aa8988-7dd9-4897-8139-1ef5005ee015","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753924} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5203138 29ca147e-169a-4e8e-b00f-3ce52224a98a /tidb/cdc/default/__cdc_meta__/owner/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5203138 29ca147e-169a-4e8e-b00f-3ce52224a98a' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/29ca147e-169a-4e8e-b00f-3ce52224a98a {"id":"29ca147e-169a-4e8e-b00f-3ce52224a98a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753928} /tidb/cdc/default/__cdc_meta__/capture/36aa8988-7dd9-4897-8139-1ef5005ee015 {"id":"36aa8988-7dd9-4897-8139-1ef5005ee015","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753924} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5203138 29ca147e-169a-4e8e-b00f-3ce52224a98a /tidb/cdc/default/__cdc_meta__/owner/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5203138 29ca147e-169a-4e8e-b00f-3ce52224a98a' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' ++ cdc cli changefeed create --pd=http://127.0.0.1:2379 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' ++ tail -n2 ++ head -n1 ++ awk '{print $2}' + changefeed_id=1738748712547619951717617538736451025585 + run_sql 'CREATE DATABASE capture_suicide_while_balance_table;' 127.0.0.1 4000 ++ seq 1 4 + for i in $(seq 1 4) + run_sql 'CREATE table capture_suicide_while_balance_table.t1 (id int primary key auto_increment)' 127.0.0.1 4000 + for i in $(seq 1 4) + run_sql 'CREATE table capture_suicide_while_balance_table.t2 (id int primary key auto_increment)' 127.0.0.1 4000 + for i in $(seq 1 4) + run_sql 'CREATE table capture_suicide_while_balance_table.t3 (id int primary key auto_increment)' 127.0.0.1 4000 + for i in $(seq 1 4) + run_sql 'CREATE table capture_suicide_while_balance_table.t4 (id int primary key auto_increment)' 127.0.0.1 4000 ++ seq 1 4 + for i in $(seq 1 4) + check_table_exists capture_suicide_while_balance_table.t1 127.0.0.1 3306 table capture_suicide_while_balance_table.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:05:30 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1065ca6f-249a-4296-b27b-24fc0821ea42 {"id":"1065ca6f-249a-4296-b27b-24fc0821ea42","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753927} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd522cde6 1065ca6f-249a-4296-b27b-24fc0821ea42 /tidb/cdc/default/__cdc_meta__/owner/223195ffd522cde6 1065ca6f-249a-4296-b27b-24fc0821ea42' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1065ca6f-249a-4296-b27b-24fc0821ea42 {"id":"1065ca6f-249a-4296-b27b-24fc0821ea42","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753927} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd522cde6 1065ca6f-249a-4296-b27b-24fc0821ea42 /tidb/cdc/default/__cdc_meta__/owner/223195ffd522cde6 1065ca6f-249a-4296-b27b-24fc0821ea42' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1065ca6f-249a-4296-b27b-24fc0821ea42 {"id":"1065ca6f-249a-4296-b27b-24fc0821ea42","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753927} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd522cde6 1065ca6f-249a-4296-b27b-24fc0821ea42 /tidb/cdc/default/__cdc_meta__/owner/223195ffd522cde6 1065ca6f-249a-4296-b27b-24fc0821ea42' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.cli.2671.out cli changefeed create --start-ts=457114629214830594 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 211048387278579274317792383677797767591 Info: {"upstream_id":7489366045077596167,"id":"211048387278579274317792383677797767591","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:05:30.899939883+08:00","start_ts":457114629214830594,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114629214830594,"checkpoint_ts":457114629214830594,"checkpoint_time":"2025-04-04 16:05:26.143","gid":{"low":2110483872785792743,"high":17792383677797767591}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x + set +x table capture_suicide_while_balance_table.t1 exists + for i in $(seq 1 4) + check_table_exists capture_suicide_while_balance_table.t2 127.0.0.1 3306 table capture_suicide_while_balance_table.t2 exists + for i in $(seq 1 4) + check_table_exists capture_suicide_while_balance_table.t3 127.0.0.1 3306 table capture_suicide_while_balance_table.t3 exists + for i in $(seq 1 4) + check_table_exists capture_suicide_while_balance_table.t4 127.0.0.1 3306 table capture_suicide_while_balance_table.t4 exists ++ cdc cli capture list ++ grep -v 'Command to ticdc' ++ jq -r '.[]|select(.address=="127.0.0.1:8300")|.id' + capture1_id=36aa8988-7dd9-4897-8139-1ef5005ee015 ++ cdc cli capture list ++ grep -v 'Command to ticdc' ++ jq -r '.[]|select(.address=="127.0.0.1:8301")|.id' + capture2_id=29ca147e-169a-4e8e-b00f-3ce52224a98a + target_capture=36aa8988-7dd9-4897-8139-1ef5005ee015 ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/1738748712547619951717617538736451025585/tables ++ jq -r --arg cid 29ca147e-169a-4e8e-b00f-3ce52224a98a '.items[] | select(.node_id==$cid) | .table_ids[0]' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 169 100 169 0 0 14083 0 --:--:-- --:--:-- --:--:-- 14083 + one_table_id=116 + [[ 116 == \n\u\l\l ]] + [[ 116 == \0 ]] ++ mysql -h127.0.0.1 -P4000 -uroot -e 'select table_name from information_schema.tables where tidb_table_id = 116\G' + table_query='*************************** 1. row *************************** table_name: t2' ++ echo cdc1.log cdc2.log cdc_data1 cdc_data2 down_pd down_pd.log oom_record pd-config.toml pd1 pd1.log sql_res.capture_suicide_while_balance_table.log stdout1.log stdout2.log tidb-config-1743753917993309577.toml tidb-config-1743753918000584952.toml tidb-config-1743753918155828529.toml tidb-config.toml tidb.log tidb_down.log tidb_other.log tiflash tiflash-config.toml tiflash-proxy.toml tikv-config.toml tikv1 tikv1.log tikv2 tikv2.log tikv3 tikv3.log tikv_down tikv_down.log 1. row cdc1.log cdc2.log cdc_data1 cdc_data2 down_pd down_pd.log oom_record pd-config.toml pd1 pd1.log sql_res.capture_suicide_while_balance_table.log stdout1.log stdout2.log tidb-config-1743753917993309577.toml tidb-config-1743753918000584952.toml tidb-config-1743753918155828529.toml tidb-config.toml tidb.log tidb_down.log tidb_other.log tiflash tiflash-config.toml tiflash-proxy.toml tikv-config.toml tikv1 tikv1.log tikv2 tikv2.log tikv3 tikv3.log tikv_down tikv_down.log table_name: t2 ++ tail -n 1 ++ awk '{print $(NF)}' + table_name=t2 + run_sql 'insert into capture_suicide_while_balance_table.t2 values (),(),(),(),()' + sleep 2 table foreign_key.finish_mark not exists for 1-th check, retry later table foreign_key.finish_mark exists check diff failed 1-th time, retry later + check_table_exists charset_gbk_test0.t0 127.0.0.1 3306 table charset_gbk_test0.t0 exists + check_table_exists charset_gbk_test0.t1 127.0.0.1 3306 table charset_gbk_test0.t1 exists + check_table_exists charset_gbk_test1.t0 127.0.0.1 3306 table charset_gbk_test1.t0 exists + check_table_exists charset_gbk_test1.t1 127.0.0.1 3306 table charset_gbk_test1.t1 not exists for 1-th check, retry later + curl -X POST 'http://127.0.0.1:8300/api/v2/changefeeds/1738748712547619951717617538736451025585/move_table?tableID=116&targetNodeID=36aa8988-7dd9-4897-8139-1ef5005ee015' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later table charset_gbk_test1.t1 exists + check_table_exists test.finish_mark 127.0.0.1 3306 table test.finish_mark not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2 100 2 0 0 1 0 0:00:02 0:00:01 0:00:01 1 100 2 100 2 0 0 1 0 0:00:02 0:00:01 0:00:01 1 {}+ sleep 2 check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/foreign_key/stdout.log no DATA RACE found [Fri Apr 4 16:05:37 CST 2025] <<<<<< run test case foreign_key success! >>>>>> check diff failed 2-th time, retry later Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 6.44 secs (617070603 bytes/sec) [Pipeline] { [Pipeline] cache table test.finish_mark not exists for 2-th check, retry later ++ ETCDCTL_API=3 ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture/29ca147e-169a-4e8e-b00f-3ce52224a98a -w json ++ grep -o 'lease":[0-9]*' ++ awk -F: '{print $2}' + lease=2463915397173227832 ++ printf '%x\n' 2463915397173227832 + lease_hex=223195ffd5203138 + ETCDCTL_API=3 + etcdctl lease revoke 223195ffd5203138 lease 223195ffd5203138 revoked + sleep 10 table test.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/vector/stdout.log no DATA RACE found [Fri Apr 4 16:05:39 CST 2025] <<<<<< run test case vector success! >>>>>> check diff successfully check diff failed 1-th time, retry later table test.finish_mark exists + echo 'check table exists success' check table exists success + check_sync_diff /tmp/tidb_cdc_test/charset_gbk /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/charset_gbk/conf/diff_config.toml 60 check diff successfully + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + check_logs /tmp/tidb_cdc_test/charset_gbk log files: /tmp/tidb_cdc_test/charset_gbk/stdout.log no DATA RACE found ++ date + echo '[Fri Apr 4 16:05:41 CST 2025] <<<<<< run test case charset_gbk success! >>>>>>' [Fri Apr 4 16:05:41 CST 2025] <<<<<< run test case charset_gbk success! >>>>>> + stop_tidb_cluster check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/new_ci_collation/stdout.log no DATA RACE found [Fri Apr 4 16:05:42 CST 2025] <<<<<< run test case new_ci_collation success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 8.17 secs (486741779 bytes/sec) [Pipeline] { [Pipeline] cache + run_cdc_server --workdir /tmp/tidb_cdc_test/capture_suicide_while_balance_table --binary cdc.test --logsuffix 2 --addr 127.0.0.1:8301 [Fri Apr 4 16:05:48 CST 2025] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/downstreamadapter/sink/mysql/MySQLSinkHangLongTime=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.28962898.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/simple/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:05:51 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3686 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36aa8988-7dd9-4897-8139-1ef5005ee015 {"id":"36aa8988-7dd9-4897-8139-1ef5005ee015","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753924} /tidb/cdc/default/__cdc_meta__/capture/4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8 {"id":"4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753948} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5203470 4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5203470 4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8 /tidb/cdc/default/default/changefeed/info/1738748712547619951717617538736451025585 {"id":{"id":{"low":17387487125476199517,"high":17617538736451025585},"display":{"name":"1738748712547619951717617538736451025585","namespace":"default"}},"upstream-id":7489366036167173390,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:05:30.78683532+08:00","start-ts":457114630428819458,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114630428819464} /tidb/cdc/default/default/changefeed/status/1738748712547619951717617538736451025585 {"checkpoint-ts":457114634439622663,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36aa8988-7dd9-4897-8139-1ef5005ee015 {"id":"36aa8988-7dd9-4897-8139-1ef5005ee015","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753924} /tidb/cdc/default/__cdc_meta__/capture/4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8 {"id":"4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753948} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5203470 4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5203470 4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8 /tidb/cdc/default/default/changefeed/info/1738748712547619951717617538736451025585 {"id":{"id":{"low":17387487125476199517,"high":17617538736451025585},"display":{"name":"1738748712547619951717617538736451025585","namespace":"default"}},"upstream-id":7489366036167173390,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:05:30.78683532+08:00","start-ts":457114630428819458,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114630428819464} /tidb/cdc/default/default/changefeed/status/1738748712547619951717617538736451025585 {"checkpoint-ts":457114634439622663,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36aa8988-7dd9-4897-8139-1ef5005ee015 {"id":"36aa8988-7dd9-4897-8139-1ef5005ee015","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753924} /tidb/cdc/default/__cdc_meta__/capture/4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8 {"id":"4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753948} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5203470 4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd52030d9 36aa8988-7dd9-4897-8139-1ef5005ee015 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5203470 4d4db590-91a1-4d1c-af0e-e27b8a8e4cf8 /tidb/cdc/default/default/changefeed/info/1738748712547619951717617538736451025585 {"id":{"id":{"low":17387487125476199517,"high":17617538736451025585},"display":{"name":"1738748712547619951717617538736451025585","namespace":"default"}},"upstream-id":7489366036167173390,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:05:30.78683532+08:00","start-ts":457114630428819458,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114630428819464} /tidb/cdc/default/default/changefeed/status/1738748712547619951717617538736451025585 {"checkpoint-ts":457114634439622663,"progress":1}' + grep -q 'etcd info' + break + set +x + check_sync_diff /tmp/tidb_cdc_test/capture_suicide_while_balance_table /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/capture_suicide_while_balance_table/conf/diff_config.toml check diff successfully + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + check_logs /tmp/tidb_cdc_test/capture_suicide_while_balance_table log files: /tmp/tidb_cdc_test/capture_suicide_while_balance_table/stdout1.log /tmp/tidb_cdc_test/capture_suicide_while_balance_table/stdout2.log no DATA RACE found ++ date + echo '[Fri Apr 4 16:05:52 CST 2025] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>>' [Fri Apr 4 16:05:52 CST 2025] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/safe_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/safe_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 8.64 secs (460120193 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + ./tests/integration_tests/run_light_it_in_ci.sh mysql G06 [Pipeline] sh Starting Upstream TiDB... Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G06 Group Number (parsed): 06 Run cases: http_api http_api_tls fail_over_ddl_G GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-3mwl6 HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=ceabf1e8-d1be-41e8-9cc7-c81c38890049 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G06 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h pingcap_ticdc_pull_cdc_mysql_integration_light_358-3mwl6 PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/http_api/run.sh using Sink-Type: mysql... <<================= + ./tests/integration_tests/run_light_it_in_ci.sh mysql G08 [Pipeline] sh Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G08 Group Number (parsed): 08 Run cases: capture_session_done_during_task changefeed_dup_error_restart mysql_sink_retry fail_over_ddl_I GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-c2312 HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=55542cfe-16d8-472f-9ee1-9577363655cf GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G08 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_ticdc_pull_cdc_mysql_integration_light_358-c2312 pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + ./tests/integration_tests/run_light_it_in_ci.sh mysql G09 [Pipeline] sh Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G09 Group Number (parsed): 09 Run cases: cdc_server_tips ddl_sequence server_config_compatibility fail_over_ddl_J GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-90s3w HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=d376fbf9-1f2d-46c5-ac5e-2161040d4629 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G09 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8 pingcap_ticdc_pull_cdc_mysql_integration_light_358-90s3w PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + ./tests/integration_tests/run_light_it_in_ci.sh mysql G10 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366204204291268 TiDB Cluster ID. tikv_gc_leader_uuid 657ff576490000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:6213, start at 2025-04-04 16:05:59.731954699 +0800 CST m=+3.367366105 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:59.740 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:59.742 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:59.742 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366204204291268 TiDB Cluster ID. tikv_gc_leader_uuid 657ff576490000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:6213, start at 2025-04-04 16:05:59.731954699 +0800 CST m=+3.367366105 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:59.740 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:59.742 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:59.742 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366202193114374 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5768000009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:6291, start at 2025-04-04 16:05:59.9518868 +0800 CST m=+3.416044805 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:07:59.961 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:05:59.936 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:55:59.936 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G10 Group Number (parsed): 10 Run cases: changefeed_error bdr_mode fail_over_ddl_K GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-bjkgt HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=7888c3f9-a742-407c-95cd-74c1231d541a GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G10 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_ticdc_pull_cdc_mysql_integration_light_358-bjkgt pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + ./tests/integration_tests/run_light_it_in_ci.sh mysql G07 requests installed. Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G07 Group Number (parsed): 07 Run cases: http_api_tls_with_user_auth fail_over_ddl_H GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-qqkbz HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=675ed5dc-3374-411f-9234-7a596596f386 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G07 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_ticdc_pull_cdc_mysql_integration_light_358-qqkbz pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366208015251980 TiDB Cluster ID. tikv_gc_leader_uuid 657ff576ca0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:3741, start at 2025-04-04 16:06:00.256228302 +0800 CST m=+3.053348342 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:00.263 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:00.266 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:00.266 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366208015251980 TiDB Cluster ID. tikv_gc_leader_uuid 657ff576ca0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:3741, start at 2025-04-04 16:06:00.256228302 +0800 CST m=+3.053348342 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:00.263 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:00.266 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:00.266 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366201470219564 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5771c40009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:3823, start at 2025-04-04 16:06:00.573786173 +0800 CST m=+3.206833892 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:00.583 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:00.561 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:00.561 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) requests installed. The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { table ci_collation_compatibility.t exists table ci_collation_compatibility.finish_mark_1 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/ci_collation_compatibility/stdout.log no DATA RACE found [Fri Apr 4 16:06:01 CST 2025] <<<<<< run test case ci_collation_compatibility success! >>>>>> [Fri Apr 4 16:06:02 CST 2025] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.75797581.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.5033.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366218843997032 TiDB Cluster ID. tikv_gc_leader_uuid 657ff579eb00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:3831, start at 2025-04-04 16:06:03.472508109 +0800 CST m=+3.079662406 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:03.480 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:03.536 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:03.536 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366218843997032 TiDB Cluster ID. tikv_gc_leader_uuid 657ff579eb00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:3831, start at 2025-04-04 16:06:03.472508109 +0800 CST m=+3.079662406 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:03.480 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:03.536 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:03.536 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366215023339255 TiDB Cluster ID. tikv_gc_leader_uuid 657ff57a360000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:3892, start at 2025-04-04 16:06:03.769452016 +0800 CST m=+3.210961222 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:03.776 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:03.776 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:03.776 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/safe_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/safe_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/safe_mode/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/safe_mode/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/safe_mode/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls_with_user_auth Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x + tso='457114639182069761 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114639182069761 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:06 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e32ed77-2e8e-48ee-bdde-387ec910917c {"id":"9e32ed77-2e8e-48ee-bdde-387ec910917c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753963} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5b60fd9 9e32ed77-2e8e-48ee-bdde-387ec910917c /tidb/cdc/default/__cdc_meta__/owner/223195ffd5b60fd9 9e32ed77-2e8e-48ee-bdde-387ec910917c' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e32ed77-2e8e-48ee-bdde-387ec910917c {"id":"9e32ed77-2e8e-48ee-bdde-387ec910917c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753963} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5b60fd9 9e32ed77-2e8e-48ee-bdde-387ec910917c /tidb/cdc/default/__cdc_meta__/owner/223195ffd5b60fd9 9e32ed77-2e8e-48ee-bdde-387ec910917c' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e32ed77-2e8e-48ee-bdde-387ec910917c {"id":"9e32ed77-2e8e-48ee-bdde-387ec910917c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753963} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5b60fd9 9e32ed77-2e8e-48ee-bdde-387ec910917c /tidb/cdc/default/__cdc_meta__/owner/223195ffd5b60fd9 9e32ed77-2e8e-48ee-bdde-387ec910917c' + grep -q 'etcd info' + break + set +x table changefeed_pause_resume.t1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366217850765779 TiDB Cluster ID. tikv_gc_leader_uuid 657ff57c0f0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:3795, start at 2025-04-04 16:06:05.661962765 +0800 CST m=+3.573786519 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:05.669 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:05.728 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:05.728 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366217850765779 TiDB Cluster ID. tikv_gc_leader_uuid 657ff57c0f0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:3795, start at 2025-04-04 16:06:05.661962765 +0800 CST m=+3.573786519 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:05.669 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:05.728 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:05.728 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366222079405235 TiDB Cluster ID. tikv_gc_leader_uuid 657ff57c278000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:3866, start at 2025-04-04 16:06:05.744802842 +0800 CST m=+3.501152699 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:05.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:05.726 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:05.726 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... [Fri Apr 4 16:06:06 CST 2025] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.50815083.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Fri Apr 4 16:06:08 CST 2025] <<<<<< START cdc server in safe_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.safe_mode.51935195.out server --log-file /tmp/tidb_cdc_test/safe_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/safe_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 exists Run 1 test Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Fri Apr 4 16:06:08 CST 2025] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.50795081.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:09 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1523bcc1-7886-4ec3-8d88-bafe4879dcb9 {"id":"1523bcc1-7886-4ec3-8d88-bafe4879dcb9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753966} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5bd2ffb 1523bcc1-7886-4ec3-8d88-bafe4879dcb9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5bd2ffb 1523bcc1-7886-4ec3-8d88-bafe4879dcb9' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1523bcc1-7886-4ec3-8d88-bafe4879dcb9 {"id":"1523bcc1-7886-4ec3-8d88-bafe4879dcb9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753966} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5bd2ffb 1523bcc1-7886-4ec3-8d88-bafe4879dcb9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5bd2ffb 1523bcc1-7886-4ec3-8d88-bafe4879dcb9' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1523bcc1-7886-4ec3-8d88-bafe4879dcb9 {"id":"1523bcc1-7886-4ec3-8d88-bafe4879dcb9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753966} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5bd2ffb 1523bcc1-7886-4ec3-8d88-bafe4879dcb9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5bd2ffb 1523bcc1-7886-4ec3-8d88-bafe4879dcb9' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.5145.out cli changefeed create --start-ts=457114639182069761 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: 492175646328138895612588933916353144740 Info: {"upstream_id":7489366208015251980,"id":"492175646328138895612588933916353144740","namespace":"default","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:06:09.959257805+08:00","start_ts":457114639182069761,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114639182069761,"checkpoint_ts":457114639182069761,"checkpoint_time":"2025-04-04 16:06:04.165","gid":{"low":4921756463281388956,"high":12588933916353144740}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + set +x succeed to verify meta placement rules Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:11 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73ec3cd9-478c-4d33-a683-8b2c21e4d510 {"id":"73ec3cd9-478c-4d33-a683-8b2c21e4d510","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753968} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5c98cf5 73ec3cd9-478c-4d33-a683-8b2c21e4d510 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5c98cf5 73ec3cd9-478c-4d33-a683-8b2c21e4d510' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73ec3cd9-478c-4d33-a683-8b2c21e4d510 {"id":"73ec3cd9-478c-4d33-a683-8b2c21e4d510","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753968} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5c98cf5 73ec3cd9-478c-4d33-a683-8b2c21e4d510 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5c98cf5 73ec3cd9-478c-4d33-a683-8b2c21e4d510' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73ec3cd9-478c-4d33-a683-8b2c21e4d510 {"id":"73ec3cd9-478c-4d33-a683-8b2c21e4d510","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753968} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5c98cf5 73ec3cd9-478c-4d33-a683-8b2c21e4d510 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5c98cf5 73ec3cd9-478c-4d33-a683-8b2c21e4d510' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.safe_mode.cli.5253.out cli changefeed create '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?safe-mode=true' Create changefeed successfully! ID: 1306070332039557596510351929465232742334 Info: {"upstream_id":7489366218843997032,"id":"1306070332039557596510351929465232742334","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-04T16:06:11.525751612+08:00","start_ts":457114641099653124,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114641099653124,"checkpoint_ts":457114641099653124,"checkpoint_time":"2025-04-04 16:06:11.480","gid":{"low":13060703320395575965,"high":10351929465232742334}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:11 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f985042a-0db0-46ff-8c50-09e6d47d884d {"id":"f985042a-0db0-46ff-8c50-09e6d47d884d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753969} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5cc45d6 f985042a-0db0-46ff-8c50-09e6d47d884d /tidb/cdc/default/__cdc_meta__/owner/223195ffd5cc45d6 f985042a-0db0-46ff-8c50-09e6d47d884d' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f985042a-0db0-46ff-8c50-09e6d47d884d {"id":"f985042a-0db0-46ff-8c50-09e6d47d884d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753969} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5cc45d6 f985042a-0db0-46ff-8c50-09e6d47d884d /tidb/cdc/default/__cdc_meta__/owner/223195ffd5cc45d6 f985042a-0db0-46ff-8c50-09e6d47d884d' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f985042a-0db0-46ff-8c50-09e6d47d884d {"id":"f985042a-0db0-46ff-8c50-09e6d47d884d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753969} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5cc45d6 f985042a-0db0-46ff-8c50-09e6d47d884d /tidb/cdc/default/__cdc_meta__/owner/223195ffd5cc45d6 f985042a-0db0-46ff-8c50-09e6d47d884d' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.cli.5145.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 6.28 secs (633616557 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check data successfully check diff failed 3-th time, retry later + set +x + tso='457114641256153094 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114641256153094 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366257097169450 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5837f0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:3900, start at 2025-04-04 16:06:13.267220693 +0800 CST m=+3.257113797 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:13.275 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:13.344 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:13.344 +0800 All versions after safe point can be accessed. (DO NOT EDIT) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/simple/stdout.log no DATA RACE found [Fri Apr 4 16:06:14 CST 2025] <<<<<< run test case simple success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366260450528404 TiDB Cluster ID. tikv_gc_leader_uuid 657ff58327c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:1289, start at 2025-04-04 16:06:12.943075136 +0800 CST m=+3.299957864 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:12.950 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:12.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:12.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366260450528404 TiDB Cluster ID. tikv_gc_leader_uuid 657ff58327c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:1289, start at 2025-04-04 16:06:12.943075136 +0800 CST m=+3.299957864 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:12.950 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:12.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:12.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366259077972826 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5831000003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:1368, start at 2025-04-04 16:06:12.802748727 +0800 CST m=+2.987545885 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:12.809 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:12.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:12.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366257097169450 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5837f0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:3900, start at 2025-04-04 16:06:13.267220693 +0800 CST m=+3.257113797 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:13.275 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:13.344 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:13.344 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366259484893880 TiDB Cluster ID. tikv_gc_leader_uuid 657ff583d24000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:3967, start at 2025-04-04 16:06:13.591999513 +0800 CST m=+3.427894104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:13.598 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:13.577 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:13.577 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366261190761405 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5844240010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:1275, start at 2025-04-04 16:06:14.058328265 +0800 CST m=+3.666654233 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:14.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:14.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:14.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366261190761405 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5844240010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:1275, start at 2025-04-04 16:06:14.058328265 +0800 CST m=+3.666654233 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:14.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:14.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:14.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366258441905208 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5849800005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:1349, start at 2025-04-04 16:06:14.376251926 +0800 CST m=+3.822634518 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:14.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:14.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:14.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366253040053092 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5845e00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:1214, start at 2025-04-04 16:06:14.145033208 +0800 CST m=+3.599550668 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:14.226 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:14.216 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:14.216 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366253040053092 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5845e00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:1214, start at 2025-04-04 16:06:14.145033208 +0800 CST m=+3.599550668 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:14.226 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:14.216 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:14.216 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366262677867623 TiDB Cluster ID. tikv_gc_leader_uuid 657ff58475c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:1304, start at 2025-04-04 16:06:14.241216538 +0800 CST m=+3.515564885 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:14.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:14.316 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:14.316 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366267078774413 TiDB Cluster ID. tikv_gc_leader_uuid 657ff585794000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:1235, start at 2025-04-04 16:06:15.291948449 +0800 CST m=+3.528483896 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:15.299 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:15.269 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:15.269 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366267078774413 TiDB Cluster ID. tikv_gc_leader_uuid 657ff585794000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:1235, start at 2025-04-04 16:06:15.291948449 +0800 CST m=+3.528483896 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:15.299 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:15.269 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:15.269 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366256761646481 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5857a0000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:1307, start at 2025-04-04 16:06:15.289563112 +0800 CST m=+3.410049517 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:15.298 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:15.322 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:15.322 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... [Fri Apr 4 16:06:16 CST 2025] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/logservice/logpuller/InjectForceReconnect=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.52325234.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.2600.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.2586.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.2538.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366265253890182 TiDB Cluster ID. tikv_gc_leader_uuid 657ff585d240033 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:1370, start at 2025-04-04 16:06:15.641652204 +0800 CST m=+4.295285501 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:15.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:15.725 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:15.725 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366265253890182 TiDB Cluster ID. tikv_gc_leader_uuid 657ff585d240033 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:1370, start at 2025-04-04 16:06:15.641652204 +0800 CST m=+4.295285501 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:15.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:15.725 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:15.725 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366253514470842 TiDB Cluster ID. tikv_gc_leader_uuid 657ff585e640004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:1425, start at 2025-04-04 16:06:15.731676533 +0800 CST m=+4.199380747 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:15.739 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:15.705 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:15.705 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_api/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='457114642492948481 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114642492948481 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... check diff successfully + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:06:18 CST 2025] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/downstreamadapter/dispatchermanager/NewEventDispatcherManagerDelay=sleep(4000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.26492651.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls_with_user_auth Starting TLS PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114642631884801 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114642631884801 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "scanproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "requestdistribution"="uniform" "mysql.port"="4000" "readallfields"="true" "threadcount"="4" "mysql.db"="changefeed_error" "updateproportion"="0" "mysql.user"="root" "insertproportion"="0" "readproportion"="0" "dotransactions"="false" "recordcount"="20" "operationcount"="0" ********************************************** Run finished, takes 10.768403ms INSERT - Takes(s): 0.0, Count: 19, OPS: 3226.4, Avg(us): 1886, Min(us): 1010, Max(us): 4886, 95th(us): 5000, 99th(us): 5000 [Fri Apr 4 16:06:18 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/logservice/schemastore/getAllPhysicalTablesGCFastFail=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.25892591.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114642674089985 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114642674089985 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Fri Apr 4 16:06:18 CST 2025] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.26262628.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting TLS TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:19 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9aa50e09-1090-4d9b-91b7-d980cb35b118 {"id":"9aa50e09-1090-4d9b-91b7-d980cb35b118","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753976} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5ed45d9 9aa50e09-1090-4d9b-91b7-d980cb35b118 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5ed45d9 9aa50e09-1090-4d9b-91b7-d980cb35b118' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9aa50e09-1090-4d9b-91b7-d980cb35b118 {"id":"9aa50e09-1090-4d9b-91b7-d980cb35b118","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753976} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5ed45d9 9aa50e09-1090-4d9b-91b7-d980cb35b118 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5ed45d9 9aa50e09-1090-4d9b-91b7-d980cb35b118' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9aa50e09-1090-4d9b-91b7-d980cb35b118 {"id":"9aa50e09-1090-4d9b-91b7-d980cb35b118","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753976} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5ed45d9 9aa50e09-1090-4d9b-91b7-d980cb35b118 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5ed45d9 9aa50e09-1090-4d9b-91b7-d980cb35b118' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:06:19 CST 2025] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.26152617.out server --log-file /tmp/tidb_cdc_test/http_api/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting TLS TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:21 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d9a33340-fea2-4bfd-9444-8c41518901dd {"id":"d9a33340-fea2-4bfd-9444-8c41518901dd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753978} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5e9d5fb d9a33340-fea2-4bfd-9444-8c41518901dd /tidb/cdc/default/__cdc_meta__/owner/223195ffd5e9d5fb d9a33340-fea2-4bfd-9444-8c41518901dd' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d9a33340-fea2-4bfd-9444-8c41518901dd {"id":"d9a33340-fea2-4bfd-9444-8c41518901dd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753978} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5e9d5fb d9a33340-fea2-4bfd-9444-8c41518901dd /tidb/cdc/default/__cdc_meta__/owner/223195ffd5e9d5fb d9a33340-fea2-4bfd-9444-8c41518901dd' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d9a33340-fea2-4bfd-9444-8c41518901dd {"id":"d9a33340-fea2-4bfd-9444-8c41518901dd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753978} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5e9d5fb d9a33340-fea2-4bfd-9444-8c41518901dd /tidb/cdc/default/__cdc_meta__/owner/223195ffd5e9d5fb d9a33340-fea2-4bfd-9444-8c41518901dd' + grep -q 'etcd info' + break + set +x cdc pid: 2654 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:21 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4e46ab0-d62b-467e-85b3-1c783f51fd5d {"id":"c4e46ab0-d62b-467e-85b3-1c783f51fd5d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753979} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaedf3 c4e46ab0-d62b-467e-85b3-1c783f51fd5d /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eaedf3 c4e46ab0-d62b-467e-85b3-1c783f51fd5d' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4e46ab0-d62b-467e-85b3-1c783f51fd5d {"id":"c4e46ab0-d62b-467e-85b3-1c783f51fd5d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753979} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaedf3 c4e46ab0-d62b-467e-85b3-1c783f51fd5d /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eaedf3 c4e46ab0-d62b-467e-85b3-1c783f51fd5d' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4e46ab0-d62b-467e-85b3-1c783f51fd5d {"id":"c4e46ab0-d62b-467e-85b3-1c783f51fd5d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753979} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaedf3 c4e46ab0-d62b-467e-85b3-1c783f51fd5d /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eaedf3 c4e46ab0-d62b-467e-85b3-1c783f51fd5d' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.2660.out cli changefeed create --start-ts=457114642631884801 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:21 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e50e654-72bf-480c-bdf5-d7ef7a7d86c6 {"id":"4e50e654-72bf-480c-bdf5-d7ef7a7d86c6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753979} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eb5bf0 4e50e654-72bf-480c-bdf5-d7ef7a7d86c6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eb5bf0 4e50e654-72bf-480c-bdf5-d7ef7a7d86c6' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e50e654-72bf-480c-bdf5-d7ef7a7d86c6 {"id":"4e50e654-72bf-480c-bdf5-d7ef7a7d86c6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753979} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eb5bf0 4e50e654-72bf-480c-bdf5-d7ef7a7d86c6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eb5bf0 4e50e654-72bf-480c-bdf5-d7ef7a7d86c6' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e50e654-72bf-480c-bdf5-d7ef7a7d86c6 {"id":"4e50e654-72bf-480c-bdf5-d7ef7a7d86c6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753979} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eb5bf0 4e50e654-72bf-480c-bdf5-d7ef7a7d86c6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eb5bf0 4e50e654-72bf-480c-bdf5-d7ef7a7d86c6' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7489366261190761405,"id":"changefeed-error","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T16:06:22.223943961+08:00","start_ts":457114642631884801,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114642631884801,"checkpoint_ts":457114642631884801,"checkpoint_time":"2025-04-04 16:06:17.325","gid":{"low":16592180046262035037,"high":9057015964442250662}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... lease 223195ffd5e9d5fb revoked command: ! ps -p 2654 PID TTY TIME CMD 2654 ? 00:00:01 cdc.test run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:22 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7506947a-ae32-4e86-a98b-dd5259f575ff {"id":"7506947a-ae32-4e86-a98b-dd5259f575ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753980} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7506947a-ae32-4e86-a98b-dd5259f575ff {"id":"7506947a-ae32-4e86-a98b-dd5259f575ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753980} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7506947a-ae32-4e86-a98b-dd5259f575ff {"id":"7506947a-ae32-4e86-a98b-dd5259f575ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753980} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff' + grep -q 'etcd info' + break + set +x command: cdc.test cli capture list 2>&1 | grep '"is_owner": true' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366298684430498 TiDB Cluster ID. tikv_gc_leader_uuid 657ff58bf380006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:3702, start at 2025-04-04 16:06:21.911186245 +0800 CST m=+3.267746458 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:21.919 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:21.902 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:21.902 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366298684430498 TiDB Cluster ID. tikv_gc_leader_uuid 657ff58bf380006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:3702, start at 2025-04-04 16:06:21.911186245 +0800 CST m=+3.267746458 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:21.919 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:21.902 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:21.902 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366297838988580 TiDB Cluster ID. tikv_gc_leader_uuid 657ff58bd94000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:3772, start at 2025-04-04 16:06:21.812236113 +0800 CST m=+3.004432034 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:21.819 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:21.882 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:21.882 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x Start case 1: Test unretryable error command: check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c changefeed-error -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366261190761405, "id": "changefeed-error", "namespace": "default", "state": "failed", "checkpoint_tso": 457114642631884801, "checkpoint_time": "2025-04-04 16:06:17.325", "error": { "time": "2025-04-04T16:06:22.470408815+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114484506624000)" } }' + echo '{ "upstream_id": 7489366261190761405, "id": "changefeed-error", "namespace": "default", "state": "failed", "checkpoint_tso": 457114642631884801, "checkpoint_time": "2025-04-04 16:06:17.325", "error": { "time": "2025-04-04T16:06:22.470408815+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114484506624000)" } }' { "upstream_id": 7489366261190761405, "id": "changefeed-error", "namespace": "default", "state": "failed", "checkpoint_tso": 457114642631884801, "checkpoint_time": "2025-04-04 16:06:17.325", "error": { "time": "2025-04-04T16:06:22.470408815+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114484506624000)" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7489366261190761405, '"id":' '"changefeed-error",' '"namespace":' '"default",' '"state":' '"failed",' '"checkpoint_tso":' 457114642631884801, '"checkpoint_time":' '"2025-04-04' '16:06:17.325",' '"error":' '{' '"time":' '"2025-04-04T16:06:22.470408815+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts '%!d(string=snapTs' %d is smaller than gcTs '%d)' is earlier than or equal to GC safepoint at '0%!(EXTRA' 'uint64=457114484506624000)"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489366261190761405, '"id":' '"changefeed-error",' '"namespace":' '"default",' '"state":' '"failed",' '"checkpoint_tso":' 457114642631884801, '"checkpoint_time":' '"2025-04-04' '16:06:17.325",' '"error":' '{' '"time":' '"2025-04-04T16:06:22.470408815+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts '%!d(string=snapTs' %d is smaller than gcTs '%d)' is earlier than or equal to GC safepoint at '0%!(EXTRA' 'uint64=457114484506624000)"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114484506624000)' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114484506624000) =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.2732.out cli changefeed resume -c changefeed-error Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 8.25 secs (481873790 bytes/sec) [Pipeline] { [Pipeline] cache PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366310674189548 TiDB Cluster ID. tikv_gc_leader_uuid 657ff58cee0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:2843, start at 2025-04-04 16:06:22.927955909 +0800 CST m=+1.427158939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:22.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:22.904 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:22.904 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Fri Apr 4 16:06:23 CST 2025] <<<<<< START cdc server in http_api_tls_with_user_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates ']' + GO_FAILPOINTS= + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls_with_user_auth.28842886.out server --log-file /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_http_api_tls_with_user_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_data_http_api_tls_with_user_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls_with_user_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... coverage: 2.3% of statements in github.com/pingcap/ticdc/... command: ! ps -p 2654 PID TTY TIME CMD 2654 ? 00:00:01 cdc.test run task failed 2-th time, retry later "is_owner": true, run task successfully + set +x table changefeed_error.usertable exists check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.5044.out cli tso query --pd=http://127.0.0.1:2379 owner pid: 2620 owner id 7506947a-ae32-4e86-a98b-dd5259f575ff default [Fri Apr 4 16:06:25 CST 2025] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.27412743.out server --log-file /tmp/tidb_cdc_test/http_api/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114644788543490 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114644788543490 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * ALPN, offering h2 * ALPN, offering http/1.1 * successfully set certificate verify locations: * CAfile: /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem CApath: none } [5 bytes data] * TLSv1.3 (OUT), TLS handshake, Client hello (1): } [512 bytes data] * TLSv1.3 (IN), TLS handshake, Server hello (2): { [122 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): { [6 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Request CERT (13): { [142 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Certificate (11): { [928 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, CERT verify (15): { [264 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Finished (20): { [36 bytes data] * TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Certificate (11): } [1814 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, CERT verify (15): } [264 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Finished (20): } [36 bytes data] * SSL connection using TLSv1.3 / TLS_AES_128_GCM_SHA256 * ALPN, server did not agree to a protocol * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * subjectAltName: host "127.0.0.1" matched cert's IP address! * issuer: C=CN; ST=Beijing; L=Beijing; O=PingCAP; CN=My own CA * SSL certificate verify ok. * Server auth using Basic with user 'ticdc' } [5 bytes data] * TLSv1.3 (OUT), TLS app data, [no content] (0): } [1 bytes data] > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > { [5 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): { [2834 bytes data] * TLSv1.3 (IN), TLS app data, [no content] (0): { [1 bytes data] < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:26 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 {"id":"61f5151e-1ef9-44ff-807c-bb6d8d0b7f46","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753984} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 {"id":"61f5151e-1ef9-44ff-807c-bb6d8d0b7f46","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753984} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 {"id":"61f5151e-1ef9-44ff-807c-bb6d8d0b7f46","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753984} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "workload"="core" "mysql.user"="root" "updateproportion"="0" "dotransactions"="false" "scanproportion"="0" "insertproportion"="0" "readproportion"="0" "mysql.db"="multi_capture_1" "threadcount"="2" "requestdistribution"="uniform" "recordcount"="10" "readallfields"="true" "mysql.port"="4000" "operationcount"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 12.613528ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1223.6, Avg(us): 2448, Min(us): 1228, Max(us): 4343, 95th(us): 5000, 99th(us): 5000 check diff successfully ***************** properties ***************** "mysql.port"="4000" "threadcount"="4" "insertproportion"="0" "workload"="core" "mysql.user"="root" "readallfields"="true" "operationcount"="0" "recordcount"="20" "mysql.host"="127.0.0.1" "dotransactions"="false" "readproportion"="0" "mysql.db"="changefeed_error" "updateproportion"="0" "scanproportion"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 4.67653ms INSERT - Takes(s): 0.0, Count: 20, OPS: 5810.8, Avg(us): 733, Min(us): 394, Max(us): 1562, 95th(us): 2000, 99th(us): 2000 check diff successfully start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit command: check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully command: check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully Pass case 1 Start case 2: Test retryable error [Fri Apr 4 16:06:28 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/maintainer/NewChangefeedRetryError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.29252927.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 command: ! ps -p 2654 PID TTY TIME CMD run task successfully cdc server already exit [Fri Apr 4 16:06:28 CST 2025] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/downstreamadapter/dispatchermanager/NewEventDispatcherManagerDelay=sleep(4000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.28092811.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 4 16:06:28 CST 2025] <<<<<< START cdc server in http_api_tls_with_user_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls_with_user_auth.29712973.out server --log-file /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_http_api_tls_with_user_auth_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_data_http_api_tls_with_user_auth_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls_with_user_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:28 GMT < Content-Length: 1184 < Content-Type: text/plain; charset=utf-8 < { [1184 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7506947a-ae32-4e86-a98b-dd5259f575ff {"id":"7506947a-ae32-4e86-a98b-dd5259f575ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753980} /tidb/cdc/default/__cdc_meta__/capture/8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 {"id":"8d85a794-d0e2-4a0e-8648-7e11cca7a5c9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753986} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7506947a-ae32-4e86-a98b-dd5259f575ff {"id":"7506947a-ae32-4e86-a98b-dd5259f575ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753980} /tidb/cdc/default/__cdc_meta__/capture/8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 {"id":"8d85a794-d0e2-4a0e-8648-7e11cca7a5c9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753986} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7506947a-ae32-4e86-a98b-dd5259f575ff {"id":"7506947a-ae32-4e86-a98b-dd5259f575ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753980} /tidb/cdc/default/__cdc_meta__/capture/8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 {"id":"8d85a794-d0e2-4a0e-8648-7e11cca7a5c9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753986} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9' + grep -q 'etcd info' + break + set +x command: cdc.test cli capture list 2>&1 | grep -v "7506947a-ae32-4e86-a98b-dd5259f575ff default" | grep id ***************** properties ***************** "workload"="core" "readallfields"="true" "mysql.host"="127.0.0.1" "dotransactions"="false" "threadcount"="2" "mysql.db"="multi_capture_2" "operationcount"="0" "scanproportion"="0" "requestdistribution"="uniform" "recordcount"="10" "mysql.user"="root" "insertproportion"="0" "readproportion"="0" "mysql.port"="4000" "updateproportion"="0" ********************************************** Run finished, takes 9.359838ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2017.9, Avg(us): 1777, Min(us): 1050, Max(us): 4298, 95th(us): 5000, 99th(us): 5000 "id": "8d85a794-d0e2-4a0e-8648-7e11cca7a5c9", run task successfully ***************** properties ***************** "scanproportion"="0" "mysql.db"="multi_capture_3" "dotransactions"="false" "requestdistribution"="uniform" "operationcount"="0" "mysql.user"="root" "workload"="core" "updateproportion"="0" "threadcount"="2" "recordcount"="10" "readallfields"="true" "mysql.port"="4000" "insertproportion"="0" "mysql.host"="127.0.0.1" "readproportion"="0" ********************************************** Run finished, takes 10.150128ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1850.2, Avg(us): 1886, Min(us): 1034, Max(us): 5361, 95th(us): 6000, 99th(us): 6000 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ***************** properties ***************** "readallfields"="true" "readproportion"="0" "dotransactions"="false" "updateproportion"="0" "insertproportion"="0" "mysql.user"="root" "workload"="core" "mysql.host"="127.0.0.1" "mysql.db"="multi_capture_4" "threadcount"="2" "mysql.port"="4000" "scanproportion"="0" "requestdistribution"="uniform" "recordcount"="10" "operationcount"="0" ********************************************** Run finished, takes 9.526699ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2021.3, Avg(us): 1810, Min(us): 994, Max(us): 4493, 95th(us): 5000, 99th(us): 5000 [Fri Apr 4 16:06:31 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.51665168.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 capture_id: 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3279 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfe41aa0-4f8c-41f8-b0ff-1aa38435473e {"id":"cfe41aa0-4f8c-41f8-b0ff-1aa38435473e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753988} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaedf3 c4e46ab0-d62b-467e-85b3-1c783f51fd5d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaef0b cfe41aa0-4f8c-41f8-b0ff-1aa38435473e /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eaef0b cfe41aa0-4f8c-41f8-b0ff-1aa38435473e /tidb/cdc/default/default/changefeed/info/changefeed-error {"id":{"id":{"low":16592180046262035037,"high":9057015964442250662},"display":{"name":"changefeed-error","namespace":"default"}},"upstream-id":7489366261190761405,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:06:22.223943961+08:00","start-ts":457114642631884801,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"warning","error":{"time":"2025-04-04T16:06:29.643504632+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114644387987465} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457114643011993612,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfe41aa0-4f8c-41f8-b0ff-1aa38435473e {"id":"cfe41aa0-4f8c-41f8-b0ff-1aa38435473e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753988} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaedf3 c4e46ab0-d62b-467e-85b3-1c783f51fd5d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaef0b cfe41aa0-4f8c-41f8-b0ff-1aa38435473e /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eaef0b cfe41aa0-4f8c-41f8-b0ff-1aa38435473e /tidb/cdc/default/default/changefeed/info/changefeed-error {"id":{"id":{"low":16592180046262035037,"high":9057015964442250662},"display":{"name":"changefeed-error","namespace":"default"}},"upstream-id":7489366261190761405,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:06:22.223943961+08:00","start-ts":457114642631884801,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"warning","error":{"time":"2025-04-04T16:06:29.643504632+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114644387987465} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457114643011993612,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfe41aa0-4f8c-41f8-b0ff-1aa38435473e {"id":"cfe41aa0-4f8c-41f8-b0ff-1aa38435473e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753988} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaedf3 c4e46ab0-d62b-467e-85b3-1c783f51fd5d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaef0b cfe41aa0-4f8c-41f8-b0ff-1aa38435473e /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eaef0b cfe41aa0-4f8c-41f8-b0ff-1aa38435473e /tidb/cdc/default/default/changefeed/info/changefeed-error {"id":{"id":{"low":16592180046262035037,"high":9057015964442250662},"display":{"name":"changefeed-error","namespace":"default"}},"upstream-id":7489366261190761405,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:06:22.223943961+08:00","start-ts":457114642631884801,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"warning","error":{"time":"2025-04-04T16:06:29.643504632+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114644387987465} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457114643011993612,"progress":1}' + grep -q 'etcd info' + break + set +x command: check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c changefeed-error -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366261190761405, "id": "changefeed-error", "namespace": "default", "state": "warning", "checkpoint_tso": 457114643011993612, "checkpoint_time": "2025-04-04 16:06:18.775", "error": { "time": "2025-04-04T16:06:29.643504632+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7489366261190761405, "id": "changefeed-error", "namespace": "default", "state": "warning", "checkpoint_tso": 457114643011993612, "checkpoint_time": "2025-04-04 16:06:18.775", "error": { "time": "2025-04-04T16:06:29.643504632+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7489366261190761405, "id": "changefeed-error", "namespace": "default", "state": "warning", "checkpoint_tso": 457114643011993612, "checkpoint_time": "2025-04-04 16:06:18.775", "error": { "time": "2025-04-04T16:06:29.643504632+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7489366261190761405, '"id":' '"changefeed-error",' '"namespace":' '"default",' '"state":' '"warning",' '"checkpoint_tso":' 457114643011993612, '"checkpoint_time":' '"2025-04-04' '16:06:18.775",' '"error":' '{' '"time":' '"2025-04-04T16:06:29.643504632+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7489366261190761405, '"id":' '"changefeed-error",' '"namespace":' '"default",' '"state":' '"warning",' '"checkpoint_tso":' 457114643011993612, '"checkpoint_time":' '"2025-04-04' '16:06:18.775",' '"error":' '{' '"time":' '"2025-04-04T16:06:29.643504632+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.3029.out cli changefeed create --start-ts=457114642631884801 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3106 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a8206f9-aa53-418f-b360-b9e61882b0cb {"id":"9a8206f9-aa53-418f-b360-b9e61882b0cb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753988} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5e9d6de 9a8206f9-aa53-418f-b360-b9e61882b0cb /tidb/cdc/default/__cdc_meta__/owner/223195ffd5e9d6de 9a8206f9-aa53-418f-b360-b9e61882b0cb /tidb/cdc/default/default/changefeed/info/1652010057082417090111232575187738903685 {"id":{"id":{"low":16520100570824170901,"high":11232575187738903685},"display":{"name":"1652010057082417090111232575187738903685","namespace":"default"}},"upstream-id":7489366260450528404,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:06:21.321614294+08:00","start-ts":457114642492948481,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114643672334344} /tidb/cdc/default/default/changefeed/status/1652010057082417090111232575187738903685 {"checkpoint-ts":457114644563623942,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a8206f9-aa53-418f-b360-b9e61882b0cb {"id":"9a8206f9-aa53-418f-b360-b9e61882b0cb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753988} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5e9d6de 9a8206f9-aa53-418f-b360-b9e61882b0cb /tidb/cdc/default/__cdc_meta__/owner/223195ffd5e9d6de 9a8206f9-aa53-418f-b360-b9e61882b0cb /tidb/cdc/default/default/changefeed/info/1652010057082417090111232575187738903685 {"id":{"id":{"low":16520100570824170901,"high":11232575187738903685},"display":{"name":"1652010057082417090111232575187738903685","namespace":"default"}},"upstream-id":7489366260450528404,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:06:21.321614294+08:00","start-ts":457114642492948481,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114643672334344} /tidb/cdc/default/default/changefeed/status/1652010057082417090111232575187738903685 {"checkpoint-ts":457114644563623942,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a8206f9-aa53-418f-b360-b9e61882b0cb {"id":"9a8206f9-aa53-418f-b360-b9e61882b0cb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753988} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5e9d6de 9a8206f9-aa53-418f-b360-b9e61882b0cb /tidb/cdc/default/__cdc_meta__/owner/223195ffd5e9d6de 9a8206f9-aa53-418f-b360-b9e61882b0cb /tidb/cdc/default/default/changefeed/info/1652010057082417090111232575187738903685 {"id":{"id":{"low":16520100570824170901,"high":11232575187738903685},"display":{"name":"1652010057082417090111232575187738903685","namespace":"default"}},"upstream-id":7489366260450528404,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:06:21.321614294+08:00","start-ts":457114642492948481,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114643672334344} /tidb/cdc/default/default/changefeed/status/1652010057082417090111232575187738903685 {"checkpoint-ts":457114644563623942,"progress":1}' + grep -q 'etcd info' + break + set +x cdc server restart success 2025-04-04 16:06:31 - INFO - Start to run test case: check_health 2025-04-04 16:06:31 - INFO - Test case check_health finished 2025-04-04 16:06:31 - INFO - Start to run test case: get_status 2025-04-04 16:06:31 - INFO - Test case get_status finished Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7489366261190761405,"id":"changefeed-error-2","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T16:06:31.831805401+08:00","start_ts":457114642631884801,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114642631884801,"checkpoint_ts":457114642631884801,"checkpoint_time":"2025-04-04 16:06:17.325","gid":{"low":522447167218370642,"high":4671799422732718989}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * ALPN, offering h2 * ALPN, offering http/1.1 * successfully set certificate verify locations: * CAfile: /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem CApath: none } [5 bytes data] * TLSv1.3 (OUT), TLS handshake, Client hello (1): } [512 bytes data] * TLSv1.3 (IN), TLS handshake, Server hello (2): { [122 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): { [6 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Request CERT (13): { [142 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Certificate (11): { [928 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, CERT verify (15): { [264 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Finished (20): { [36 bytes data] * TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Certificate (11): } [1814 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, CERT verify (15): } [264 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Finished (20): } [36 bytes data] * SSL connection using TLSv1.3 / TLS_AES_128_GCM_SHA256 * ALPN, server did not agree to a protocol * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * subjectAltName: host "127.0.0.1" matched cert's IP address! * issuer: C=CN; ST=Beijing; L=Beijing; O=PingCAP; CN=My own CA * SSL certificate verify ok. * Server auth using Basic with user 'ticdc' } [5 bytes data] * TLSv1.3 (OUT), TLS app data, [no content] (0): } [1 bytes data] > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > { [5 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): { [2834 bytes data] * TLSv1.3 (IN), TLS app data, [no content] (0): { [1 bytes data] < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:31 GMT < Content-Length: 1184 < Content-Type: text/plain; charset=utf-8 < { [1067 bytes data] * TLSv1.3 (IN), TLS app data, [no content] (0): { [1 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10bd617e-045f-4caf-8896-625c80843d5b {"id":"10bd617e-045f-4caf-8896-625c80843d5b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753989} /tidb/cdc/default/__cdc_meta__/capture/61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 {"id":"61f5151e-1ef9-44ff-807c-bb6d8d0b7f46","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753984} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6224b67 10bd617e-045f-4caf-8896-625c80843d5b /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6224b67 10bd617e-045f-4caf-8896-625c80843d5b' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10bd617e-045f-4caf-8896-625c80843d5b {"id":"10bd617e-045f-4caf-8896-625c80843d5b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753989} /tidb/cdc/default/__cdc_meta__/capture/61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 {"id":"61f5151e-1ef9-44ff-807c-bb6d8d0b7f46","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753984} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6224b67 10bd617e-045f-4caf-8896-625c80843d5b /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6224b67 10bd617e-045f-4caf-8896-625c80843d5b' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10bd617e-045f-4caf-8896-625c80843d5b {"id":"10bd617e-045f-4caf-8896-625c80843d5b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753989} /tidb/cdc/default/__cdc_meta__/capture/61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 {"id":"61f5151e-1ef9-44ff-807c-bb6d8d0b7f46","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753984} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6224b67 10bd617e-045f-4caf-8896-625c80843d5b /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6224ade 61f5151e-1ef9-44ff-807c-bb6d8d0b7f46 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6224b67 10bd617e-045f-4caf-8896-625c80843d5b' + grep -q 'etcd info' + break + set +x 2025-04-04 16:06:32 - INFO - Start to run test case: create_changefeed 2025-04-04 16:06:32 - INFO - Start to post request, URL: http://127.0.0.1:8301/api/v2/changefeeds 2025-04-04 16:06:32 - INFO - Request body: {"changefeed_id": "changefeed-test1", "sink_uri": "mysql://normal:123456@127.0.0.1:3306/", "ignore_ineligible_table": true} 2025-04-04 16:06:32 - INFO - Response status code: 200 2025-04-04 16:06:32 - INFO - Response body: {'upstream_id': 7489366265253890182, 'id': 'changefeed-test1', 'namespace': 'default', 'sink_uri': 'mysql://normal:xxxxx@127.0.0.1:3306/', 'create_time': '2025-04-04T16:06:32.023136494+08:00', 'start_ts': 457114646482780161, 'config': {'memory_quota': 1073741824, 'case_sensitive': False, 'force_replicate': False, 'ignore_ineligible_table': False, 'check_gc_safe_point': True, 'enable_sync_point': False, 'enable_table_monitor': False, 'bdr_mode': False, 'sync_point_interval': 600000000000, 'sync_point_retention': 86400000000000, 'filter': {'rules': ['*.*']}, 'mounter': {'worker_num': 16}, 'sink': {'csv': {'delimiter': ',', 'quote': '"', 'null': '\\N', 'include_commit_ts': False, 'binary_encoding_method': 'base64', 'output_old_value': False, 'output_handle_key': False}, 'encoder_concurrency': 32, 'terminator': '\r\n', 'date_separator': 'day', 'enable_partition_separator': True, 'enable_kafka_sink_v2': False, 'only_output_updated_columns': False, 'delete_only_output_handle_key_columns': False, 'content_compatible': False, 'advance_timeout': 150, 'send_bootstrap_interval_in_sec': 120, 'send_bootstrap_in_msg_count': 10000, 'send_bootstrap_to_all_partition': True, 'send-all-bootstrap-at-start': False, 'debezium_disable_schema': False, 'debezium': {'output_old_value': True}, 'open': {'output_old_value': True}}, 'consistent': {'level': 'none', 'max_log_size': 64, 'flush_interval': 2000, 'meta_flush_interval': 200, 'encoding_worker_num': 16, 'flush_worker_num': 8, 'use_file_backend': False, 'memory_usage': {'memory_quota_percentage': 50}}, 'scheduler': {'enable_table_across_nodes': False, 'region_threshold': 100000, 'write_key_threshold': 0, 'split_number_per_node': 1}, 'integrity': {'integrity_check_level': 'none', 'corruption_handle_level': 'warn'}, 'changefeed_error_stuck_duration': 1800000000000, 'synced_status': {'synced_check_interval': 300, 'checkpoint_interval': 15}}, 'state': 'normal', 'creator_version': 'v9.0.0-beta.1-55-g52a1009d', 'resolved_ts': 457114646482780161, 'checkpoint_ts': 457114646482780161, 'checkpoint_time': '2025-04-04 16:06:32.015', 'gid': {'low': 8378427588254890057, 'high': 17388613550528634517}} 2025-04-04 16:06:32 - INFO - Start to post request, URL: http://127.0.0.1:8301/api/v2/changefeeds 2025-04-04 16:06:32 - INFO - Request body: {"changefeed_id": "changefeed-test2", "sink_uri": "blackhole://", "ignore_ineligible_table": true} 2025-04-04 16:06:32 - INFO - Response status code: 200 2025-04-04 16:06:32 - INFO - Response body: {'upstream_id': 7489366265253890182, 'id': 'changefeed-test2', 'namespace': 'default', 'sink_uri': 'blackhole:', 'create_time': '2025-04-04T16:06:32.048781958+08:00', 'start_ts': 457114646482780171, 'config': {'memory_quota': 1073741824, 'case_sensitive': False, 'force_replicate': False, 'ignore_ineligible_table': False, 'check_gc_safe_point': True, 'enable_sync_point': False, 'enable_table_monitor': False, 'bdr_mode': False, 'sync_point_interval': 600000000000, 'sync_point_retention': 86400000000000, 'filter': {'rules': ['*.*']}, 'mounter': {'worker_num': 16}, 'sink': {'csv': {'delimiter': ',', 'quote': '"', 'null': '\\N', 'include_commit_ts': False, 'binary_encoding_method': 'base64', 'output_old_value': False, 'output_handle_key': False}, 'encoder_concurrency': 32, 'terminator': '\r\n', 'date_separator': 'day', 'enable_partition_separator': True, 'enable_kafka_sink_v2': False, 'only_output_updated_columns': False, 'delete_only_output_handle_key_columns': False, 'content_compatible': False, 'advance_timeout': 150, 'send_bootstrap_interval_in_sec': 120, 'send_bootstrap_in_msg_count': 10000, 'send_bootstrap_to_all_partition': True, 'send-all-bootstrap-at-start': False, 'debezium_disable_schema': False, 'debezium': {'output_old_value': True}, 'open': {'output_old_value': True}}, 'consistent': {'level': 'none', 'max_log_size': 64, 'flush_interval': 2000, 'meta_flush_interval': 200, 'encoding_worker_num': 16, 'flush_worker_num': 8, 'use_file_backend': False, 'memory_usage': {'memory_quota_percentage': 50}}, 'scheduler': {'enable_table_across_nodes': False, 'region_threshold': 100000, 'write_key_threshold': 0, 'split_number_per_node': 1}, 'integrity': {'integrity_check_level': 'none', 'corruption_handle_level': 'warn'}, 'changefeed_error_stuck_duration': 1800000000000, 'synced_status': {'synced_check_interval': 300, 'checkpoint_interval': 15}}, 'state': 'normal', 'creator_version': 'v9.0.0-beta.1-55-g52a1009d', 'resolved_ts': 457114646482780171, 'checkpoint_ts': 457114646482780171, 'checkpoint_time': '2025-04-04 16:06:32.015', 'gid': {'low': 5928627049998660130, 'high': 526059487113872271}} 2025-04-04 16:06:32 - INFO - Start to post request, URL: http://127.0.0.1:8301/api/v2/changefeeds 2025-04-04 16:06:32 - INFO - Request body: {"changefeed_id": "changefeed-test3", "sink_uri": "blackhole://", "ignore_ineligible_table": true} 2025-04-04 16:06:32 - INFO - Response status code: 200 2025-04-04 16:06:32 - INFO - Response body: {'upstream_id': 7489366265253890182, 'id': 'changefeed-test3', 'namespace': 'default', 'sink_uri': 'blackhole:', 'create_time': '2025-04-04T16:06:32.063104514+08:00', 'start_ts': 457114646482780177, 'config': {'memory_quota': 1073741824, 'case_sensitive': False, 'force_replicate': False, 'ignore_ineligible_table': False, 'check_gc_safe_point': True, 'enable_sync_point': False, 'enable_table_monitor': False, 'bdr_mode': False, 'sync_point_interval': 600000000000, 'sync_point_retention': 86400000000000, 'filter': {'rules': ['*.*']}, 'mounter': {'worker_num': 16}, 'sink': {'csv': {'delimiter': ',', 'quote': '"', 'null': '\\N', 'include_commit_ts': False, 'binary_encoding_method': 'base64', 'output_old_value': False, 'output_handle_key': False}, 'encoder_concurrency': 32, 'terminator': '\r\n', 'date_separator': 'day', 'enable_partition_separator': True, 'enable_kafka_sink_v2': False, 'only_output_updated_columns': False, 'delete_only_output_handle_key_columns': False, 'content_compatible': False, 'advance_timeout': 150, 'send_bootstrap_interval_in_sec': 120, 'send_bootstrap_in_msg_count': 10000, 'send_bootstrap_to_all_partition': True, 'send-all-bootstrap-at-start': False, 'debezium_disable_schema': False, 'debezium': {'output_old_value': True}, 'open': {'output_old_value': True}}, 'consistent': {'level': 'none', 'max_log_size': 64, 'flush_interval': 2000, 'meta_flush_interval': 200, 'encoding_worker_num': 16, 'flush_worker_num': 8, 'use_file_backend': False, 'memory_usage': {'memory_quota_percentage': 50}}, 'scheduler': {'enable_table_across_nodes': False, 'region_threshold': 100000, 'write_key_threshold': 0, 'split_number_per_node': 1}, 'integrity': {'integrity_check_level': 'none', 'corruption_handle_level': 'warn'}, 'changefeed_error_stuck_duration': 1800000000000, 'synced_status': {'synced_check_interval': 300, 'checkpoint_interval': 15}}, 'state': 'normal', 'creator_version': 'v9.0.0-beta.1-55-g52a1009d', 'resolved_ts': 457114646482780177, 'checkpoint_ts': 457114646482780177, 'checkpoint_time': '2025-04-04 16:06:32.015', 'gid': {'low': 7152189326328481213, 'high': 13651011388923813554}} 2025-04-04 16:06:32 - INFO - response status code: 400 2025-04-04 16:06:32 - INFO - response body: {'error_msg': "[CDC:ErrSinkURIInvalid]sink uri invalid '%s': [CDC:ErrMySQLConnectionError]fail to open MySQL connection: dial tcp 127.0.0.1:1111: connect: connection refused", 'error_code': 'CDC:ErrSinkURIInvalid'} 2025-04-04 16:06:32 - INFO - Test case create_changefeed finished Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table capture_session_done_during_task.t exists check diff successfully check diff failed 1-th time, retry later Run 2 test + set +x command: check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c changefeed-error-2 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366261190761405, "id": "changefeed-error-2", "namespace": "default", "state": "warning", "checkpoint_tso": 457114642631884801, "checkpoint_time": "2025-04-04 16:06:17.325", "error": { "time": "2025-04-04T16:06:32.04365897+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7489366261190761405, "id": "changefeed-error-2", "namespace": "default", "state": "warning", "checkpoint_tso": 457114642631884801, "checkpoint_time": "2025-04-04 16:06:17.325", "error": { "time": "2025-04-04T16:06:32.04365897+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7489366261190761405, "id": "changefeed-error-2", "namespace": "default", "state": "warning", "checkpoint_tso": 457114642631884801, "checkpoint_time": "2025-04-04 16:06:17.325", "error": { "time": "2025-04-04T16:06:32.04365897+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7489366261190761405, '"id":' '"changefeed-error-2",' '"namespace":' '"default",' '"state":' '"warning",' '"checkpoint_tso":' 457114642631884801, '"checkpoint_time":' '"2025-04-04' '16:06:17.325",' '"error":' '{' '"time":' '"2025-04-04T16:06:32.04365897+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7489366261190761405, '"id":' '"changefeed-error-2",' '"namespace":' '"default",' '"state":' '"warning",' '"checkpoint_tso":' 457114642631884801, '"checkpoint_time":' '"2025-04-04' '16:06:17.325",' '"error":' '{' '"time":' '"2025-04-04T16:06:32.04365897+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.3102.out cli changefeed remove -c changefeed-error check diff failed 1-th time, retry later Changefeed remove successfully. ID: changefeed-error CheckpointTs: 457114643011993612 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.3% of statements in github.com/pingcap/ticdc/... Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 7.89 secs (504276866 bytes/sec) [Pipeline] { [Pipeline] cache pass test: check health pass test: get status + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:34 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/673015db-ca2c-45fa-ace3-f7f4f0d7d19c {"id":"673015db-ca2c-45fa-ace3-f7f4f0d7d19c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753991} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/673015db-ca2c-45fa-ace3-f7f4f0d7d19c {"id":"673015db-ca2c-45fa-ace3-f7f4f0d7d19c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753991} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/673015db-ca2c-45fa-ace3-f7f4f0d7d19c {"id":"673015db-ca2c-45fa-ace3-f7f4f0d7d19c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753991} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:06:34 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.52215223.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8302 failed: Connection refused * Failed to connect to 127.0.0.1 port 8302: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: create changefeed command: check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c changefeed-test1 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366310674189548, "id": "changefeed-test1", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647056613387, "checkpoint_time": "2025-04-04 16:06:34.204", "error": null }' + echo '{ "upstream_id": 7489366310674189548, "id": "changefeed-test1", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647056613387, "checkpoint_time": "2025-04-04 16:06:34.204", "error": null }' { "upstream_id": 7489366310674189548, "id": "changefeed-test1", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647056613387, "checkpoint_time": "2025-04-04 16:06:34.204", "error": null } ++ echo '{' '"upstream_id":' 7489366310674189548, '"id":' '"changefeed-test1",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114647056613387, '"checkpoint_time":' '"2025-04-04' '16:06:34.204",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366310674189548, '"id":' '"changefeed-test1",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114647056613387, '"checkpoint_time":' '"2025-04-04' '16:06:34.204",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully command: check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c changefeed-test2 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366310674189548, "id": "changefeed-test2", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647082827779, "checkpoint_time": "2025-04-04 16:06:34.304", "error": null }' + echo '{ "upstream_id": 7489366310674189548, "id": "changefeed-test2", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647082827779, "checkpoint_time": "2025-04-04 16:06:34.304", "error": null }' { "upstream_id": 7489366310674189548, "id": "changefeed-test2", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647082827779, "checkpoint_time": "2025-04-04 16:06:34.304", "error": null } ++ echo '{' '"upstream_id":' 7489366310674189548, '"id":' '"changefeed-test2",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114647082827779, '"checkpoint_time":' '"2025-04-04' '16:06:34.304",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366310674189548, '"id":' '"changefeed-test2",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114647082827779, '"checkpoint_time":' '"2025-04-04' '16:06:34.304",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully command: check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c changefeed-test3 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366310674189548, "id": "changefeed-test3", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647095934991, "checkpoint_time": "2025-04-04 16:06:34.354", "error": null }' + echo '{ "upstream_id": 7489366310674189548, "id": "changefeed-test3", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647095934991, "checkpoint_time": "2025-04-04 16:06:34.354", "error": null }' { "upstream_id": 7489366310674189548, "id": "changefeed-test3", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647095934991, "checkpoint_time": "2025-04-04 16:06:34.354", "error": null } ++ echo '{' '"upstream_id":' 7489366310674189548, '"id":' '"changefeed-test3",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114647095934991, '"checkpoint_time":' '"2025-04-04' '16:06:34.354",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366310674189548, '"id":' '"changefeed-test3",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114647095934991, '"checkpoint_time":' '"2025-04-04' '16:06:34.354",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully command: check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c changefeed-test4 -s ++ grep -v 'Command to ticdc' + set +x command: check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.3166.out cli changefeed remove -c changefeed-error-2 + info='{ "upstream_id": 7489366310674189548, "id": "changefeed-test4", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647109042193, "checkpoint_time": "2025-04-04 16:06:34.404", "error": null }' + echo '{ "upstream_id": 7489366310674189548, "id": "changefeed-test4", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647109042193, "checkpoint_time": "2025-04-04 16:06:34.404", "error": null }' { "upstream_id": 7489366310674189548, "id": "changefeed-test4", "namespace": "default", "state": "normal", "checkpoint_tso": 457114647109042193, "checkpoint_time": "2025-04-04 16:06:34.404", "error": null } ++ echo '{' '"upstream_id":' 7489366310674189548, '"id":' '"changefeed-test4",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114647109042193, '"checkpoint_time":' '"2025-04-04' '16:06:34.404",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366310674189548, '"id":' '"changefeed-test4",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114647109042193, '"checkpoint_time":' '"2025-04-04' '16:06:34.404",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/capture_session_done_during_task/stdout.log /tmp/tidb_cdc_test/capture_session_done_during_task/stdout1-2.log no DATA RACE found [Fri Apr 4 16:06:35 CST 2025] <<<<<< run test case capture_session_done_during_task success! >>>>>> Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 457114642631884801 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.3% of statements in github.com/pingcap/ticdc/... check diff failed 2-th time, retry later table test.simple1 exists 2025-04-04 16:06:36 - INFO - Start to run test case: list_changefeed 2025-04-04 16:06:36 - INFO - Test case list_changefeed finished 2025-04-04 16:06:36 - INFO - Start to run test case: get_changefeed 2025-04-04 16:06:36 - INFO - Test case get_changefeed finished 2025-04-04 16:06:36 - INFO - Start to run test case: pause_changefeed 2025-04-04 16:06:36 - INFO - Test case pause_changefeed finished 2025-04-04 16:06:36 - INFO - Start to run test case: update_changefeed 2025-04-04 16:06:36 - INFO - Test case update_changefeed finished + set +x command: check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366355659583140 TiDB Cluster ID. tikv_gc_leader_uuid 657ff599a700006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:6240, start at 2025-04-04 16:06:35.94220871 +0800 CST m=+3.545724444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:35.950 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:35.951 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:35.951 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366355659583140 TiDB Cluster ID. tikv_gc_leader_uuid 657ff599a700006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:6240, start at 2025-04-04 16:06:35.94220871 +0800 CST m=+3.545724444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:35.950 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:35.951 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:35.951 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366356609789713 TiDB Cluster ID. tikv_gc_leader_uuid 657ff599c780002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:6316, start at 2025-04-04 16:06:36.063755319 +0800 CST m=+3.497989656 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:36.071 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:36.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:36.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2620) - No such process wait process 2620 exit for 1-th time... process 2620 already exit command: cdc.test cli capture list --server http://127.0.0.1:8301 2>&1 |grep 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 -A1 | grep '"is_owner": true' cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit command: check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully Pass case 2 Start case 3: updating GC safepoint failure case [Fri Apr 4 16:06:37 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.32693271.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8302 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:37 GMT < Content-Length: 1184 < Content-Type: text/plain; charset=utf-8 < { [1184 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/673015db-ca2c-45fa-ace3-f7f4f0d7d19c {"id":"673015db-ca2c-45fa-ace3-f7f4f0d7d19c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753991} /tidb/cdc/default/__cdc_meta__/capture/7e12ca5e-c1b8-45f6-b45c-a95694798c25 {"id":"7e12ca5e-c1b8-45f6-b45c-a95694798c25","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753994} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25 /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/673015db-ca2c-45fa-ace3-f7f4f0d7d19c {"id":"673015db-ca2c-45fa-ace3-f7f4f0d7d19c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753991} /tidb/cdc/default/__cdc_meta__/capture/7e12ca5e-c1b8-45f6-b45c-a95694798c25 {"id":"7e12ca5e-c1b8-45f6-b45c-a95694798c25","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753994} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25 /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/673015db-ca2c-45fa-ace3-f7f4f0d7d19c {"id":"673015db-ca2c-45fa-ace3-f7f4f0d7d19c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753991} /tidb/cdc/default/__cdc_meta__/capture/7e12ca5e-c1b8-45f6-b45c-a95694798c25 {"id":"7e12ca5e-c1b8-45f6-b45c-a95694798c25","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753994} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25 /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:06:37 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.52945296.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8303 failed: Connection refused * Failed to connect to 127.0.0.1 port 8303: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later table test.simple1 exists running list_changefeed pass test: list changefeed "is_owner": true, run task successfully [Fri Apr 4 16:06:38 CST 2025] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.29412943.out server --log-file /tmp/tidb_cdc_test/http_api/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.7530.out cli tso query --pd=http://127.0.0.1:2379 running get_changefeed check diff successfully pass test: get changefeed running pause_changefeed + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:40 GMT < Content-Length: 706 < Content-Type: text/plain; charset=utf-8 < { [706 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37fd3de1-ce3b-4222-a0dd-2da66e49b507 {"id":"37fd3de1-ce3b-4222-a0dd-2da66e49b507","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753997} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaef0b cfe41aa0-4f8c-41f8-b0ff-1aa38435473e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaf008 37fd3de1-ce3b-4222-a0dd-2da66e49b507 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eaf008 37fd3de1-ce3b-4222-a0dd-2da66e49b507' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37fd3de1-ce3b-4222-a0dd-2da66e49b507 {"id":"37fd3de1-ce3b-4222-a0dd-2da66e49b507","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753997} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaef0b cfe41aa0-4f8c-41f8-b0ff-1aa38435473e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaf008 37fd3de1-ce3b-4222-a0dd-2da66e49b507 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eaf008 37fd3de1-ce3b-4222-a0dd-2da66e49b507' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37fd3de1-ce3b-4222-a0dd-2da66e49b507 {"id":"37fd3de1-ce3b-4222-a0dd-2da66e49b507","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753997} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaef0b cfe41aa0-4f8c-41f8-b0ff-1aa38435473e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5eaf008 37fd3de1-ce3b-4222-a0dd-2da66e49b507 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5eaf008 37fd3de1-ce3b-4222-a0dd-2da66e49b507' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.3324.out cli changefeed create --start-ts=457114642631884801 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-3 + set +x + tso='457114648432082948 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114648432082948 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:06:40 CST 2025] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.75707572.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8303 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:40 GMT < Content-Length: 1765 < Content-Type: text/plain; charset=utf-8 < { [1765 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/673015db-ca2c-45fa-ace3-f7f4f0d7d19c {"id":"673015db-ca2c-45fa-ace3-f7f4f0d7d19c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753991} /tidb/cdc/default/__cdc_meta__/capture/7e12ca5e-c1b8-45f6-b45c-a95694798c25 {"id":"7e12ca5e-c1b8-45f6-b45c-a95694798c25","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753994} /tidb/cdc/default/__cdc_meta__/capture/e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae {"id":"e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753997} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d630 e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25 /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d630 e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/673015db-ca2c-45fa-ace3-f7f4f0d7d19c {"id":"673015db-ca2c-45fa-ace3-f7f4f0d7d19c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753991} /tidb/cdc/default/__cdc_meta__/capture/7e12ca5e-c1b8-45f6-b45c-a95694798c25 {"id":"7e12ca5e-c1b8-45f6-b45c-a95694798c25","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753994} /tidb/cdc/default/__cdc_meta__/capture/e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae {"id":"e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753997} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d630 e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25 /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d630 e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/673015db-ca2c-45fa-ace3-f7f4f0d7d19c {"id":"673015db-ca2c-45fa-ace3-f7f4f0d7d19c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753991} /tidb/cdc/default/__cdc_meta__/capture/7e12ca5e-c1b8-45f6-b45c-a95694798c25 {"id":"7e12ca5e-c1b8-45f6-b45c-a95694798c25","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753994} /tidb/cdc/default/__cdc_meta__/capture/e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae {"id":"e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753997} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd614d630 e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d53e 673015db-ca2c-45fa-ace3-f7f4f0d7d19c /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d597 7e12ca5e-c1b8-45f6-b45c-a95694798c25 /tidb/cdc/default/__cdc_meta__/owner/223195ffd614d630 e34b34b7-aa1f-4f4c-b4dd-eaf2f7ffa4ae' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.5361.out cli changefeed create --start-ts=457114644788543490 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: 1421675964567023660713426796001322187143 Info: {"upstream_id":7489366298684430498,"id":"1421675964567023660713426796001322187143","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:06:40.678582821+08:00","start_ts":457114644788543490,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114644788543490,"checkpoint_ts":457114644788543490,"checkpoint_time":"2025-04-04 16:06:25.552","gid":{"low":14216759645670236607,"high":13426796001322187143}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... pass test: pause changefeed Create changefeed successfully! ID: changefeed-error-3 Info: {"upstream_id":7489366261190761405,"id":"changefeed-error-3","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T16:06:40.908281399+08:00","start_ts":457114642631884801,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114642631884801,"checkpoint_ts":457114642631884801,"checkpoint_time":"2025-04-04 16:06:17.325","gid":{"low":13567724942262308598,"high":4568452567592153486}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... running update_changefeed pass test: update changefeed + set +x table multi_capture_1.usertable exists table multi_capture_2.usertable exists table multi_capture_3.usertable exists table multi_capture_4.usertable exists check diff failed 1-th time, retry later + set +x command: check_changefeed_state http://127.0.0.1:2379 changefeed-error-3 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-3 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c changefeed-error-3 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366261190761405, "id": "changefeed-error-3", "namespace": "default", "state": "normal", "checkpoint_tso": 457114642631884801, "checkpoint_time": "2025-04-04 16:06:17.325", "error": null }' + echo '{ "upstream_id": 7489366261190761405, "id": "changefeed-error-3", "namespace": "default", "state": "normal", "checkpoint_tso": 457114642631884801, "checkpoint_time": "2025-04-04 16:06:17.325", "error": null }' { "upstream_id": 7489366261190761405, "id": "changefeed-error-3", "namespace": "default", "state": "normal", "checkpoint_tso": 457114642631884801, "checkpoint_time": "2025-04-04 16:06:17.325", "error": null } ++ echo '{' '"upstream_id":' 7489366261190761405, '"id":' '"changefeed-error-3",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114642631884801, '"checkpoint_time":' '"2025-04-04' '16:06:17.325",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 1-th time, retry later Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 6.83 secs (582452900 bytes/sec) [Pipeline] { + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:41 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3971 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 {"id":"8d85a794-d0e2-4a0e-8648-7e11cca7a5c9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753986} /tidb/cdc/default/__cdc_meta__/capture/a2a073f8-d1ad-4dac-a178-8958efa74284 {"id":"a2a073f8-d1ad-4dac-a178-8958efa74284","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753999} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f0579f a2a073f8-d1ad-4dac-a178-8958efa74284 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f0579f a2a073f8-d1ad-4dac-a178-8958efa74284 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"id":{"id":{"low":8378427588254890057,"high":17388613550528634517},"display":{"name":"changefeed-test1","namespace":"default"}},"upstream-id":7489366265253890182,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:06:32.023136494+08:00","start-ts":457114646482780161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114646482780165} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"id":{"id":{"low":5928627049998660130,"high":526059487113872271},"display":{"name":"changefeed-test2","namespace":"default"}},"upstream-id":7489366265253890182,"sink-uri":"blackhole://","create-time":"2025-04-04T16:06:32.048781958+08:00","start-ts":457114646482780171,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114646482780173} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"id":{"id":{"low":7152189326328481213,"high":13651011388923813554},"display":{"name":"changefeed-test3","namespace":"default"}},"upstream-id":7489366265253890182,"sink-uri":"blackhole://","create-time":"2025-04-04T16:06:32.063104514+08:00","start-ts":457114646482780177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114646482780179} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":457114646508994573,"progress":1} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":457114646508994573,"progress":3} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":457114646640066613,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 {"id":"8d85a794-d0e2-4a0e-8648-7e11cca7a5c9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753986} /tidb/cdc/default/__cdc_meta__/capture/a2a073f8-d1ad-4dac-a178-8958efa74284 {"id":"a2a073f8-d1ad-4dac-a178-8958efa74284","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753999} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f0579f a2a073f8-d1ad-4dac-a178-8958efa74284 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f0579f a2a073f8-d1ad-4dac-a178-8958efa74284 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"id":{"id":{"low":8378427588254890057,"high":17388613550528634517},"display":{"name":"changefeed-test1","namespace":"default"}},"upstream-id":7489366265253890182,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:06:32.023136494+08:00","start-ts":457114646482780161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114646482780165} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"id":{"id":{"low":5928627049998660130,"high":526059487113872271},"display":{"name":"changefeed-test2","namespace":"default"}},"upstream-id":7489366265253890182,"sink-uri":"blackhole://","create-time":"2025-04-04T16:06:32.048781958+08:00","start-ts":457114646482780171,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114646482780173} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"id":{"id":{"low":7152189326328481213,"high":13651011388923813554},"display":{"name":"changefeed-test3","namespace":"default"}},"upstream-id":7489366265253890182,"sink-uri":"blackhole://","create-time":"2025-04-04T16:06:32.063104514+08:00","start-ts":457114646482780177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114646482780179} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":457114646508994573,"progress":1} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":457114646508994573,"progress":3} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":457114646640066613,"progress":1}' + grep -q 'etcd info' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 {"id":"8d85a794-d0e2-4a0e-8648-7e11cca7a5c9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753986} /tidb/cdc/default/__cdc_meta__/capture/a2a073f8-d1ad-4dac-a178-8958efa74284 {"id":"a2a073f8-d1ad-4dac-a178-8958efa74284","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743753999} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f05554 7506947a-ae32-4e86-a98b-dd5259f575ff /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd5f0579f a2a073f8-d1ad-4dac-a178-8958efa74284 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f055e4 8d85a794-d0e2-4a0e-8648-7e11cca7a5c9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd5f0579f a2a073f8-d1ad-4dac-a178-8958efa74284 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"id":{"id":{"low":8378427588254890057,"high":17388613550528634517},"display":{"name":"changefeed-test1","namespace":"default"}},"upstream-id":7489366265253890182,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:06:32.023136494+08:00","start-ts":457114646482780161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114646482780165} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"id":{"id":{"low":5928627049998660130,"high":526059487113872271},"display":{"name":"changefeed-test2","namespace":"default"}},"upstream-id":7489366265253890182,"sink-uri":"blackhole://","create-time":"2025-04-04T16:06:32.048781958+08:00","start-ts":457114646482780171,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114646482780173} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"id":{"id":{"low":7152189326328481213,"high":13651011388923813554},"display":{"name":"changefeed-test3","namespace":"default"}},"upstream-id":7489366265253890182,"sink-uri":"blackhole://","create-time":"2025-04-04T16:06:32.063104514+08:00","start-ts":457114646482780177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114646482780179} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":457114646508994573,"progress":1} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":457114646508994573,"progress":3} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":457114646640066613,"progress":1}' + break + set +x command: cdc.test cli capture list 2>&1 | grep '"address": "127.0.0.1:8300"' [Pipeline] cache "address": "127.0.0.1:8300", run task successfully 2025-04-04 16:06:43 - INFO - Start to run test case: list_changefeed 2025-04-04 16:06:43 - INFO - Test case list_changefeed finished running resume_changefeed pass test: resume changefeed 2025-04-04 16:06:43 - INFO - Start to run test case: get_changefeed 2025-04-04 16:06:43 - INFO - Test case get_changefeed finished 2025-04-04 16:06:43 - INFO - Start to run test case: pause_changefeed 2025-04-04 16:06:43 - INFO - Test case pause_changefeed finished 2025-04-04 16:06:43 - INFO - Start to run test case: resume_changefeed 2025-04-04 16:06:43 - INFO - Test case resume_changefeed finished 2025-04-04 16:06:43 - INFO - Start to run test case: move_table 2025-04-04 16:06:43 - INFO - Test case move_table finished valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Fri Apr 4 16:06:41 CST 2025] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ true != \n\o ]] + set +x + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.27272729.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None 2025-04-04 16:06:44 - INFO - Start to run test case: set_log_level 2025-04-04 16:06:44 - INFO - Test case set_log_level finished 2025-04-04 16:06:44 - INFO - Start to run test case: remove_changefeed 2025-04-04 16:06:44 - INFO - Test case remove_changefeed finished + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:06:43 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39b02f18-f5dc-48bc-a9bf-b089aa020a8f {"id":"39b02f18-f5dc-48bc-a9bf-b089aa020a8f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754001} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd646a0e6 39b02f18-f5dc-48bc-a9bf-b089aa020a8f /tidb/cdc/default/__cdc_meta__/owner/223195ffd646a0e6 39b02f18-f5dc-48bc-a9bf-b089aa020a8f' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39b02f18-f5dc-48bc-a9bf-b089aa020a8f {"id":"39b02f18-f5dc-48bc-a9bf-b089aa020a8f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754001} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd646a0e6 39b02f18-f5dc-48bc-a9bf-b089aa020a8f /tidb/cdc/default/__cdc_meta__/owner/223195ffd646a0e6 39b02f18-f5dc-48bc-a9bf-b089aa020a8f' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39b02f18-f5dc-48bc-a9bf-b089aa020a8f {"id":"39b02f18-f5dc-48bc-a9bf-b089aa020a8f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754001} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd646a0e6 39b02f18-f5dc-48bc-a9bf-b089aa020a8f /tidb/cdc/default/__cdc_meta__/owner/223195ffd646a0e6 39b02f18-f5dc-48bc-a9bf-b089aa020a8f' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.7637.out cli changefeed create --start-ts=457114648432082948 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ 2025-04-04 16:06:44 - INFO - Start to run test case: resign_owner 2025-04-04 16:06:44 - INFO - Test case resign_owner finished 2025-04-04 16:06:44 - INFO - Start to run test case: get_tso Create changefeed successfully! ID: 758701354619404013111958149916522986650 Info: {"upstream_id":7489366355659583140,"id":"758701354619404013111958149916522986650","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:06:44.294029941+08:00","start_ts":457114648432082948,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114648432082948,"checkpoint_ts":457114648432082948,"checkpoint_time":"2025-04-04 16:06:39.451","gid":{"low":7587013546194040131,"high":11958149916522986650}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... running move_table 2025-04-04 16:06:44 - INFO - Test case get_tso finished pass test: move table check diff failed 2-th time, retry later command: check_changefeed_state http://127.0.0.1:2379 changefeed-error-3 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-3 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c changefeed-error-3 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366261190761405, "id": "changefeed-error-3", "namespace": "default", "state": "failed", "checkpoint_tso": 457114643011993612, "checkpoint_time": "2025-04-04 16:06:18.775", "error": { "time": "0001-01-01T00:00:00Z", "addr": "", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114643011993612 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7489366261190761405, "id": "changefeed-error-3", "namespace": "default", "state": "failed", "checkpoint_tso": 457114643011993612, "checkpoint_time": "2025-04-04 16:06:18.775", "error": { "time": "0001-01-01T00:00:00Z", "addr": "", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114643011993612 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7489366261190761405, "id": "changefeed-error-3", "namespace": "default", "state": "failed", "checkpoint_tso": 457114643011993612, "checkpoint_time": "2025-04-04 16:06:18.775", "error": { "time": "0001-01-01T00:00:00Z", "addr": "", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114643011993612 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7489366261190761405, '"id":' '"changefeed-error-3",' '"namespace":' '"default",' '"state":' '"failed",' '"checkpoint_tso":' 457114643011993612, '"checkpoint_time":' '"2025-04-04' '16:06:18.775",' '"error":' '{' '"time":' '"0001-01-01T00:00:00Z",' '"addr":' '"",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114643011993612 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7489366261190761405, '"id":' '"changefeed-error-3",' '"namespace":' '"default",' '"state":' '"failed",' '"checkpoint_tso":' 457114643011993612, '"checkpoint_time":' '"2025-04-04' '16:06:18.775",' '"error":' '{' '"time":' '"0001-01-01T00:00:00Z",' '"addr":' '"",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114643011993612 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114643011993612 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114643011993612 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.3439.out cli changefeed remove -c changefeed-error-3 Changefeed remove successfully. ID: changefeed-error-3 CheckpointTs: 457114643011993612 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.3% of statements in github.com/pingcap/ticdc/... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/http_api/stdout.log /tmp/tidb_cdc_test/http_api/stdout1.log /tmp/tidb_cdc_test/http_api/stdout2.log no DATA RACE found [Fri Apr 4 16:06:44 CST 2025] <<<<<< run test case http_api success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... running set_log_level + set +x pass test: set log level + set +x check diff successfully ***************** properties ***************** "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "threadcount"="2" "scanproportion"="0" "readallfields"="true" "recordcount"="20" "insertproportion"="0" "readproportion"="0" "mysql.user"="root" "dotransactions"="false" "workload"="core" "mysql.port"="4000" "operationcount"="0" "mysql.db"="multi_capture_1" "updateproportion"="0" ********************************************** Run finished, takes 11.602036ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1925.8, Avg(us): 984, Min(us): 455, Max(us): 1644, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "insertproportion"="0" "scanproportion"="0" "readproportion"="0" "workload"="core" "mysql.user"="root" "updateproportion"="0" "recordcount"="20" "mysql.port"="4000" "threadcount"="2" "operationcount"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.db"="multi_capture_2" "dotransactions"="false" ********************************************** Run finished, takes 10.960903ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2017.0, Avg(us): 945, Min(us): 391, Max(us): 1579, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.host"="127.0.0.1" "workload"="core" "threadcount"="2" "scanproportion"="0" "requestdistribution"="uniform" "dotransactions"="false" "mysql.db"="multi_capture_3" "readproportion"="0" "mysql.user"="root" "mysql.port"="4000" "insertproportion"="0" "readallfields"="true" "updateproportion"="0" "recordcount"="20" "operationcount"="0" ********************************************** Run finished, takes 12.058884ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1862.1, Avg(us): 1145, Min(us): 540, Max(us): 2210, 95th(us): 2000, 99th(us): 3000 ***************** properties ***************** "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "insertproportion"="0" "recordcount"="20" "operationcount"="0" "scanproportion"="0" "readproportion"="0" "dotransactions"="false" "threadcount"="2" "updateproportion"="0" "mysql.port"="4000" "workload"="core" "readallfields"="true" "mysql.user"="root" "mysql.db"="multi_capture_4" ********************************************** Run finished, takes 11.470332ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1935.7, Avg(us): 967, Min(us): 419, Max(us): 1564, 95th(us): 2000, 99th(us): 2000 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit Pass case 3 log files: /tmp/tidb_cdc_test/changefeed_error/stdout.log no DATA RACE found [Fri Apr 4 16:06:46 CST 2025] <<<<<< run test case changefeed_error success! >>>>>> running remove_changefeed check diff failed 1-th time, retry later pass test: remove changefeed running resign_owner pass test: resign owner running get_tso pass test: get tso check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/multi_capture/stdout1.log /tmp/tidb_cdc_test/multi_capture/stdout2.log /tmp/tidb_cdc_test/multi_capture/stdout3.log no DATA RACE found [Fri Apr 4 16:06:49 CST 2025] <<<<<< run test case multi_capture success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/changefeed_dup_error_restart Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... running verify_table pass test: verify table running unsafe_apis pass test: unsafe apis Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc) 3976284160 bytes in 7.74 secs (513441979 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + ./tests/integration_tests/run_light_it_in_ci.sh mysql G13 [Pipeline] sh Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G13 Group Number (parsed): 13 Run cases: cli_tls_with_auth cli_with_auth fail_over_ddl_N GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-n1qgb HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=f3d8b2c6-2015-43b2-9780-e879bd8a4c7b GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G13 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s pingcap_ticdc_pull_cdc_mysql_integration_light_358-n1qgb PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + ./tests/integration_tests/run_light_it_in_ci.sh mysql G15 [Pipeline] sh Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G15 Group Number (parsed): 15 Run cases: split_region changefeed_resume_with_checkpoint_ts autorandom gc_safepoint GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-sddr0 HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=6885ea65-addc-4940-acc7-2e39e0f445ad GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G15 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_ticdc_pull_cdc_mysql_integration_light_358-sddr0 pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9 PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/split_region/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + ./tests/integration_tests/run_light_it_in_ci.sh mysql G14 [Pipeline] sh Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G14 Group Number (parsed): 14 Run cases: batch_add_table batch_update_to_no_batch fail_over_ddl_O GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-6mcfg HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=905c9712-fd04-49b5-9ae3-7680a427e861 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G14 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_ticdc_pull_cdc_mysql_integration_light_358-6mcfg pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/batch_add_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + ./tests/integration_tests/run_light_it_in_ci.sh mysql G11 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/http_api_tls_with_user_auth/stdout_http_api_tls_with_user_auth_tls1.log /tmp/tidb_cdc_test/http_api_tls_with_user_auth/stdout_http_api_tls_with_user_auth_tls2.log no DATA RACE found [Fri Apr 4 16:06:53 CST 2025] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G11 Group Number (parsed): 11 Run cases: multi_tables_ddl ddl_attributes multi_cdc_cluster fail_over_ddl_L GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-hbpq7 HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=70908111-1f24-4962-9112-8addc6c9f02d GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G11 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_ticdc_pull_cdc_mysql_integration_light_358-hbpq7 pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 24927 failed: Connection refused * Failed to connect to 127.0.0.1 port 24927: Connection refused * Closing connection 0 + ./tests/integration_tests/run_light_it_in_ci.sh mysql G12 Starting Upstream TiDB... Warning: Found 53 test cases that are not covered by any group: avro_basic canal_json_adapter_compatibility canal_json_basic canal_json_claim_check canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered ddl_for_split_tables debezium ds_memory_control event_filter kafka_big_messages kafka_big_messages_v2 kafka_column_selector kafka_column_selector_avro kafka_compression kafka_messages kafka_simple_basic kafka_simple_basic_avro kafka_simple_claim_check kafka_simple_claim_check_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_sink_error_resume kill_owner_with_ddl lossy_ddl mq_sink_dispatcher mq_sink_lost_callback multi_tables_ddl_v2 multi_topics multi_topics_v2 open_protocol_claim_check open_protocol_handle_key_only overwrite_resume_with_syncpoint owner_resign sequence storage_cleanup storage_csv_update synced_status synced_status_with_redo These test cases need to be added to appropriate groups in: 1. run_light_it_in_ci.sh - for light test cases 2. run_heavy_it_in_ci.sh - for heavy test cases Choose the file based on the test case's resource requirements. Sink Type: mysql Group Name: G12 Group Number (parsed): 12 Run cases: row_format tiflash multi_rocks fail_over_ddl_M GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/ticdc/pull_cdc_mysql_integration_light","buildid":"1908064800135450624","prowjobid":"6ebdb594-9052-4525-b920-f2ff078bbe64","refs":{"org":"pingcap","repo":"ticdc","repo_link":"https://github.com/pingcap/ticdc","base_ref":"master","base_sha":"5d77460e01078ac672ee4eeeef20afe573266424","base_link":"https://github.com/pingcap/ticdc/commit/5d77460e01078ac672ee4eeeef20afe573266424","pulls":[{"number":1208,"author":"dveeden","sha":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","title":"Upgrade go-mysql to v1.12.0","head_ref":"upgrade_go-mysql","link":"https://github.com/pingcap/ticdc/pull/1208","commit_link":"https://github.com/pingcap/ticdc/pull/1208/commits/52a1009d7bcd84b283c5443601c9b3160b35bb7c","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908064800135450624 POD_LABEL=pingcap_ticdc_pull_cdc_mysql_integration_light_358-fkq8d HOSTNAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=changes TICDC_NEWARCH=true JENKINS_NODE_COOKIE=120a4bfa-fbd1-4fc0-a3c1-79659210213f GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_mysql_integration_light WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=358 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light PWD=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/ticdc/pull_cdc_mysql_integration_light RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/display/redirect PROW_JOB_ID=6ebdb594-9052-4525-b920-f2ff078bbe64 TEST_GROUP=G12 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/358/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/ticdc/job/pull_cdc_mysql_integration_light/ BUILD_DISPLAY_NAME=#358 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs pingcap_ticdc_pull_cdc_mysql_integration_light_358-fkq8d PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-d32479e2efafd17bb1bb7566c145c237e9f4b88122b50a94e860dc1b03480663 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) You are running an older version of MinIO released 4 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/http_api_tls/run.sh using Sink-Type: mysql... <<================= requests installed. The 1 times to try to start tidb cluster... * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > Host: 127.0.0.1:24927 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 18330F09D6398689 < X-Xss-Protection: 1; mode=block < Date: Fri, 04 Apr 2025 08:06:55 GMT < { [226 bytes data] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/bdr_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Run 3 test check diff failed 1-th time, retry later check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366448079444258 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5af1b00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:3796, start at 2025-04-04 16:06:57.930975968 +0800 CST m=+3.893144157 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:58.002 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:58.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:58.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366448079444258 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5af1b00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:3796, start at 2025-04-04 16:06:57.930975968 +0800 CST m=+3.893144157 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:58.002 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:58.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:58.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366444061059157 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5af4e00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:3881, start at 2025-04-04 16:06:58.114663591 +0800 CST m=+3.899483831 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:08:58.122 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:06:58.104 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:56:58.104 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table safe_mode.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/safe_mode/stdout.log no DATA RACE found [Fri Apr 4 16:06:54 CST 2025] <<<<<< run test case safe_mode success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 3-th time, retry later table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.cli.5231.out cli tso query --pd=http://127.0.0.1:2379 Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table partition_table.t2 not exists for 2-th check, retry later invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Fri Apr 4 16:07:01 CST 2025] <<<<<< run all test cases cdc_server_tips success! >>>>>> check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='457114654079975428 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114654079975428 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "insertproportion"="0" "readallfields"="true" "readproportion"="0" "mysql.port"="4000" "threadcount"="2" "recordcount"="10" "operationcount"="0" "scanproportion"="0" "updateproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.db"="changefeed_dup_error_restart" "workload"="core" "requestdistribution"="uniform" "mysql.user"="root" ********************************************** Run finished, takes 10.651649ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1619.5, Avg(us): 1911, Min(us): 892, Max(us): 5992, 95th(us): 6000, 99th(us): 6000 [Fri Apr 4 16:07:02 CST 2025] <<<<<< START cdc server in changefeed_dup_error_restart case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/sink/mysql/MySQLDuplicateEntryError=5%return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.52745276.out server --log-file /tmp/tidb_cdc_test/changefeed_dup_error_restart/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_dup_error_restart/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_H/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/savepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:05 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ab59bc1-c506-455d-88be-ddfca34672b8 {"id":"2ab59bc1-c506-455d-88be-ddfca34672b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754022} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd69939eb 2ab59bc1-c506-455d-88be-ddfca34672b8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd69939eb 2ab59bc1-c506-455d-88be-ddfca34672b8' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ab59bc1-c506-455d-88be-ddfca34672b8 {"id":"2ab59bc1-c506-455d-88be-ddfca34672b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754022} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd69939eb 2ab59bc1-c506-455d-88be-ddfca34672b8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd69939eb 2ab59bc1-c506-455d-88be-ddfca34672b8' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ab59bc1-c506-455d-88be-ddfca34672b8 {"id":"2ab59bc1-c506-455d-88be-ddfca34672b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754022} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd69939eb 2ab59bc1-c506-455d-88be-ddfca34672b8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd69939eb 2ab59bc1-c506-455d-88be-ddfca34672b8' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.cli.5337.out cli changefeed create --start-ts=457114654079975428 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_H Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: 105390927182671325261029672363652655271 Info: {"upstream_id":7489366448079444258,"id":"105390927182671325261029672363652655271","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T16:07:05.887785828+08:00","start_ts":457114654079975428,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114654079975428,"checkpoint_ts":457114654079975428,"checkpoint_time":"2025-04-04 16:07:00.996","gid":{"low":10539092718267132526,"high":1029672363652655271}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366478686547938 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b78c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:1249, start at 2025-04-04 16:07:06.579801517 +0800 CST m=+3.996136629 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:06.651 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:06.639 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:06.639 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366478686547938 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b78c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:1249, start at 2025-04-04 16:07:06.579801517 +0800 CST m=+3.996136629 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:06.651 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:06.639 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:06.639 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366484778799940 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b7a50000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:1322, start at 2025-04-04 16:07:06.677717684 +0800 CST m=+3.910841842 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:06.746 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:06.744 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:06.744 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366483274802325 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b73180008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:1154, start at 2025-04-04 16:07:06.195318371 +0800 CST m=+3.412439320 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:06.204 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:06.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:06.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366483274802325 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b73180008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:1154, start at 2025-04-04 16:07:06.195318371 +0800 CST m=+3.412439320 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:06.204 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:06.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:06.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366486115439833 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b7638000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:1215, start at 2025-04-04 16:07:06.418168205 +0800 CST m=+3.505326195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:06.425 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:06.382 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:06.382 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x table changefeed_dup_error_restart.finish_mark_1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366485776363436 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b79500002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:1283, start at 2025-04-04 16:07:06.581225611 +0800 CST m=+3.800990551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:06.589 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:06.580 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:06.580 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366485776363436 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b79500002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:1283, start at 2025-04-04 16:07:06.581225611 +0800 CST m=+3.800990551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:06.589 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:06.580 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:06.580 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366482386000617 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b7c68000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:1357, start at 2025-04-04 16:07:06.803821871 +0800 CST m=+3.897881818 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:06.813 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:06.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:06.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table changefeed_dup_error_restart.finish_mark_1 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later [Fri Apr 4 16:07:09 CST 2025] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.25232525.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2555.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366498814786281 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ba1400010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:1299, start at 2025-04-04 16:07:09.165238647 +0800 CST m=+3.359310275 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:09.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:09.165 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:09.165 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.2548.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366489127912635 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b90480006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:4011, start at 2025-04-04 16:07:08.06027584 +0800 CST m=+4.485949494 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:08.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:08.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:08.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366489127912635 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b90480006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:4011, start at 2025-04-04 16:07:08.06027584 +0800 CST m=+4.485949494 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:08.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:08.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:08.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366490328606749 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b94e00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:4063, start at 2025-04-04 16:07:08.345050795 +0800 CST m=+4.597521945 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:08.354 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:08.344 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:08.344 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... table partition_table.finish_mark not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366498814786281 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ba1400010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:1299, start at 2025-04-04 16:07:09.165238647 +0800 CST m=+3.359310275 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:09.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:09.165 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:09.165 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366494269819324 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ba460000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:1362, start at 2025-04-04 16:07:09.357487952 +0800 CST m=+3.405870809 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:09.364 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:09.364 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:09.364 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Logging trace to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366488385339169 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b91240003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:1375, start at 2025-04-04 16:07:08.127169425 +0800 CST m=+3.990871952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:08.134 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:08.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:08.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366488385339169 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b91240003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:1375, start at 2025-04-04 16:07:08.127169425 +0800 CST m=+3.990871952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:08.134 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:08.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:08.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366479261050947 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5b9040000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:1437, start at 2025-04-04 16:07:08.064849481 +0800 CST m=+3.816117587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:08.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:08.098 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:08.098 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114656369803265 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114656369803265 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:11 CST 2025] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.25922594.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114656382648321 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114656382648321 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:11 CST 2025] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.25942596.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366495161243379 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ba2a40011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:4388, start at 2025-04-04 16:07:09.261496182 +0800 CST m=+4.394478848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:09.333 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:09.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:09.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366495161243379 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ba2a40011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:4388, start at 2025-04-04 16:07:09.261496182 +0800 CST m=+4.394478848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:09.333 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:09.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:09.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366492174113348 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ba4340007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:4448, start at 2025-04-04 16:07:09.338610798 +0800 CST m=+4.290917908 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:09.347 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:09.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:09.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table partition_table.finish_mark not exists for 5-th check, retry later check diff successfully ***************** properties ***************** "readproportion"="0" "scanproportion"="0" "requestdistribution"="uniform" "dotransactions"="false" "recordcount"="10" "readallfields"="true" "threadcount"="2" "insertproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "updateproportion"="0" "mysql.user"="root" "mysql.db"="changefeed_dup_error_restart" "workload"="core" ********************************************** Run finished, takes 3.899257ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3588.6, Avg(us): 681, Min(us): 458, Max(us): 1534, 95th(us): 2000, 99th(us): 2000 table changefeed_dup_error_restart.finish_mark_2 not exists for 1-th check, retry later The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366509626806843 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5bc7780013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:6521, start at 2025-04-04 16:07:11.591890903 +0800 CST m=+3.541977911 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:11.603 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:11.582 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:11.582 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366509626806843 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5bc7780013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:6521, start at 2025-04-04 16:07:11.591890903 +0800 CST m=+3.541977911 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:11.603 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:11.582 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:11.582 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366503727707937 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5bc9840006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:6580, start at 2025-04-04 16:07:11.720047378 +0800 CST m=+3.514360627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:11.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:11.713 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:11.713 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:12 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae118b57-5c94-4962-befd-e8bf84837bf6 {"id":"ae118b57-5c94-4962-befd-e8bf84837bf6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754029} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6b873d2 ae118b57-5c94-4962-befd-e8bf84837bf6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6b873d2 ae118b57-5c94-4962-befd-e8bf84837bf6' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae118b57-5c94-4962-befd-e8bf84837bf6 {"id":"ae118b57-5c94-4962-befd-e8bf84837bf6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754029} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6b873d2 ae118b57-5c94-4962-befd-e8bf84837bf6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6b873d2 ae118b57-5c94-4962-befd-e8bf84837bf6' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae118b57-5c94-4962-befd-e8bf84837bf6 {"id":"ae118b57-5c94-4962-befd-e8bf84837bf6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754029} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6b873d2 ae118b57-5c94-4962-befd-e8bf84837bf6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6b873d2 ae118b57-5c94-4962-befd-e8bf84837bf6' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.2579.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Starting Upstream TiFlash... The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting TLS PD... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Create changefeed successfully! ID: 1010709689261253931517961719890590276508 Info: {"upstream_id":7489366478686547938,"id":"1010709689261253931517961719890590276508","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:12.719767495+08:00","start_ts":457114657145225218,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114657145225218,"checkpoint_ts":457114657145225218,"checkpoint_time":"2025-04-04 16:07:12.689","gid":{"low":10107096892612539315,"high":17961719890590276508}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.cli.2631.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 4 16:07:13 CST 2025] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.56495651.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_dup_error_restart.finish_mark_2 exists check diff successfully + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + tso='457114657165410305 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114657165410305 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:14 CST 2025] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.26642666.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:14 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01303ddd-fcb5-46c2-b189-6006cb9a871b {"id":"01303ddd-fcb5-46c2-b189-6006cb9a871b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754031} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6bb26e8 01303ddd-fcb5-46c2-b189-6006cb9a871b /tidb/cdc/default/__cdc_meta__/owner/223195ffd6bb26e8 01303ddd-fcb5-46c2-b189-6006cb9a871b' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01303ddd-fcb5-46c2-b189-6006cb9a871b {"id":"01303ddd-fcb5-46c2-b189-6006cb9a871b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754031} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6bb26e8 01303ddd-fcb5-46c2-b189-6006cb9a871b /tidb/cdc/default/__cdc_meta__/owner/223195ffd6bb26e8 01303ddd-fcb5-46c2-b189-6006cb9a871b' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01303ddd-fcb5-46c2-b189-6006cb9a871b {"id":"01303ddd-fcb5-46c2-b189-6006cb9a871b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754031} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6bb26e8 01303ddd-fcb5-46c2-b189-6006cb9a871b /tidb/cdc/default/__cdc_meta__/owner/223195ffd6bb26e8 01303ddd-fcb5-46c2-b189-6006cb9a871b' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2652.out cli changefeed create --start-ts=457114656369803265 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/changefeed_dup_error_restart/stdout.log no DATA RACE found [Fri Apr 4 16:07:14 CST 2025] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> table partition_table.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:14 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5218abe5-fc4a-4d6d-86ba-551ef5647a7a {"id":"5218abe5-fc4a-4d6d-86ba-551ef5647a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754031} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6b76efc 5218abe5-fc4a-4d6d-86ba-551ef5647a7a /tidb/cdc/default/__cdc_meta__/owner/223195ffd6b76efc 5218abe5-fc4a-4d6d-86ba-551ef5647a7a' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5218abe5-fc4a-4d6d-86ba-551ef5647a7a {"id":"5218abe5-fc4a-4d6d-86ba-551ef5647a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754031} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6b76efc 5218abe5-fc4a-4d6d-86ba-551ef5647a7a /tidb/cdc/default/__cdc_meta__/owner/223195ffd6b76efc 5218abe5-fc4a-4d6d-86ba-551ef5647a7a' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5218abe5-fc4a-4d6d-86ba-551ef5647a7a {"id":"5218abe5-fc4a-4d6d-86ba-551ef5647a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754031} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6b76efc 5218abe5-fc4a-4d6d-86ba-551ef5647a7a /tidb/cdc/default/__cdc_meta__/owner/223195ffd6b76efc 5218abe5-fc4a-4d6d-86ba-551ef5647a7a' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.2650.out cli changefeed create --start-ts=457114656382648321 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.7871.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: 1529572432115503344417194141400732422588 Info: {"upstream_id":7489366483274802325,"id":"1529572432115503344417194141400732422588","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:14.543972982+08:00","start_ts":457114656369803265,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114656369803265,"checkpoint_ts":457114656369803265,"checkpoint_time":"2025-04-04 16:07:09.731","gid":{"low":15295724321155033444,"high":17194141400732422588}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... table batch_add_table.finish_mark not exists for 1-th check, retry later Create changefeed successfully! ID: 19669350874805182577189290395436937615 Info: {"upstream_id":7489366485776363436,"id":"19669350874805182577189290395436937615","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:14.662465684+08:00","start_ts":457114656382648321,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114656382648321,"checkpoint_ts":457114656382648321,"checkpoint_time":"2025-04-04 16:07:09.780","gid":{"low":1966935087480518257,"high":7189290395436937615}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366517910410119 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5befcc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:4155, start at 2025-04-04 16:07:14.1806628 +0800 CST m=+3.505248414 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:14.188 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:14.163 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:14.163 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366517910410119 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5befcc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:4155, start at 2025-04-04 16:07:14.1806628 +0800 CST m=+3.505248414 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:14.188 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:14.163 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:14.163 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366522324835705 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5befbc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:4238, start at 2025-04-04 16:07:14.167707534 +0800 CST m=+3.379285474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:14.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:14.159 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:14.159 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting TLS TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting TLS TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table split_region.test1 exists table split_region.test2 exists check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:16 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba020719-2b70-4fff-8407-dccc99b3eae0 {"id":"ba020719-2b70-4fff-8407-dccc99b3eae0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754033} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6c37be8 ba020719-2b70-4fff-8407-dccc99b3eae0 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6c37be8 ba020719-2b70-4fff-8407-dccc99b3eae0' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba020719-2b70-4fff-8407-dccc99b3eae0 {"id":"ba020719-2b70-4fff-8407-dccc99b3eae0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754033} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6c37be8 ba020719-2b70-4fff-8407-dccc99b3eae0 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6c37be8 ba020719-2b70-4fff-8407-dccc99b3eae0' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba020719-2b70-4fff-8407-dccc99b3eae0 {"id":"ba020719-2b70-4fff-8407-dccc99b3eae0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754033} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6c37be8 ba020719-2b70-4fff-8407-dccc99b3eae0 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6c37be8 ba020719-2b70-4fff-8407-dccc99b3eae0' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:07:16 CST 2025] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.57035705.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8400 failed: Connection refused * Failed to connect to 127.0.0.1 port 8400: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 7-th check, retry later + set +x + tso='457114657766506497 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114657766506497 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:16 CST 2025] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.78997901.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark exists check diff failed 1-th time, retry later Run 4 test Starting TLS TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting TLS TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:17 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bcafb873-2d82-43cb-8941-a26d7829971a {"id":"bcafb873-2d82-43cb-8941-a26d7829971a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754034} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6c34be5 bcafb873-2d82-43cb-8941-a26d7829971a /tidb/cdc/default/__cdc_meta__/owner/223195ffd6c34be5 bcafb873-2d82-43cb-8941-a26d7829971a' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bcafb873-2d82-43cb-8941-a26d7829971a {"id":"bcafb873-2d82-43cb-8941-a26d7829971a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754034} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6c34be5 bcafb873-2d82-43cb-8941-a26d7829971a /tidb/cdc/default/__cdc_meta__/owner/223195ffd6c34be5 bcafb873-2d82-43cb-8941-a26d7829971a' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bcafb873-2d82-43cb-8941-a26d7829971a {"id":"bcafb873-2d82-43cb-8941-a26d7829971a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754034} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6c34be5 bcafb873-2d82-43cb-8941-a26d7829971a /tidb/cdc/default/__cdc_meta__/owner/223195ffd6c34be5 bcafb873-2d82-43cb-8941-a26d7829971a' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7489366498814786281,"id":"test-normal","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:17.302214089+08:00","start_ts":457114657165410305,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114657165410305,"checkpoint_ts":457114657165410305,"checkpoint_time":"2025-04-04 16:07:12.766","gid":{"low":3912258857190326169,"high":7748901408240970918}} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7489366498814786281,"id":"test-error-1","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:17.419338785+08:00","start_ts":457114657165410305,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114657165410305,"checkpoint_ts":457114657165410305,"checkpoint_time":"2025-04-04 16:07:12.766","gid":{"low":8666504450883204043,"high":14712827190405623697}} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7489366498814786281,"id":"test-error-2","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:17.542692257+08:00","start_ts":457114657165410305,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114657165410305,"checkpoint_ts":457114657165410305,"checkpoint_time":"2025-04-04 16:07:12.766","gid":{"low":6867836732239149996,"high":17353760647523317180}} + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.5572.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366527657414658 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5c1b200004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:6340, start at 2025-04-04 16:07:16.942310487 +0800 CST m=+3.594711005 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:16.950 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:16.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:16.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366527657414658 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5c1b200004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:6340, start at 2025-04-04 16:07:16.942310487 +0800 CST m=+3.594711005 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:16.950 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:16.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:16.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366532165642094 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5c1b4c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:6415, start at 2025-04-04 16:07:16.980452264 +0800 CST m=+3.498051320 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:17.042 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:17.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:17.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table partition_table.finish_mark not exists for 8-th check, retry later check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366544682273763 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5c2f20000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:2915, start at 2025-04-04 16:07:18.258674049 +0800 CST m=+1.433100668 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:18.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:18.265 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:18.265 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.2966.out cli tso query --pd=https://127.0.0.1:2579 + set +x + tso='457114658422652929 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114658422652929 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:18 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.56105612.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366544095705795 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5c34280006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:5510, start at 2025-04-04 16:07:18.548448078 +0800 CST m=+1.655537485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:18.555 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:18.538 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:18.538 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Fri Apr 4 16:07:18 CST 2025] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.55525554.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8400 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:19 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/149b1e75-667a-450c-8976-c5938f2f0fce {"id":"149b1e75-667a-450c-8976-c5938f2f0fce","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754036} /tidb/cdc/default/__cdc_meta__/log_coordinator/351195ffd6c3a80a 149b1e75-667a-450c-8976-c5938f2f0fce /tidb/cdc/default/__cdc_meta__/owner/351195ffd6c3a80a 149b1e75-667a-450c-8976-c5938f2f0fce' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/149b1e75-667a-450c-8976-c5938f2f0fce {"id":"149b1e75-667a-450c-8976-c5938f2f0fce","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754036} /tidb/cdc/default/__cdc_meta__/log_coordinator/351195ffd6c3a80a 149b1e75-667a-450c-8976-c5938f2f0fce /tidb/cdc/default/__cdc_meta__/owner/351195ffd6c3a80a 149b1e75-667a-450c-8976-c5938f2f0fce' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/149b1e75-667a-450c-8976-c5938f2f0fce {"id":"149b1e75-667a-450c-8976-c5938f2f0fce","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754036} /tidb/cdc/default/__cdc_meta__/log_coordinator/351195ffd6c3a80a 149b1e75-667a-450c-8976-c5938f2f0fce /tidb/cdc/default/__cdc_meta__/owner/351195ffd6c3a80a 149b1e75-667a-450c-8976-c5938f2f0fce' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.5757.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306 -c test-1 --config=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/bdr_mode/conf/up.toml cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:19 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44aa5af8-64b6-4584-82e8-2850d6decce1 {"id":"44aa5af8-64b6-4584-82e8-2850d6decce1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754036} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6cff5e8 44aa5af8-64b6-4584-82e8-2850d6decce1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6cff5e8 44aa5af8-64b6-4584-82e8-2850d6decce1' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44aa5af8-64b6-4584-82e8-2850d6decce1 {"id":"44aa5af8-64b6-4584-82e8-2850d6decce1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754036} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6cff5e8 44aa5af8-64b6-4584-82e8-2850d6decce1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6cff5e8 44aa5af8-64b6-4584-82e8-2850d6decce1' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44aa5af8-64b6-4584-82e8-2850d6decce1 {"id":"44aa5af8-64b6-4584-82e8-2850d6decce1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754036} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6cff5e8 44aa5af8-64b6-4584-82e8-2850d6decce1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6cff5e8 44aa5af8-64b6-4584-82e8-2850d6decce1' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.7972.out cli changefeed create --start-ts=457114657766506497 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 1832413510151190960714684207395166810797 Info: {"upstream_id":7489366509626806843,"id":"1832413510151190960714684207395166810797","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:19.820586159+08:00","start_ts":457114657766506497,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114657766506497,"checkpoint_ts":457114657766506497,"checkpoint_time":"2025-04-04 16:07:15.059","gid":{"low":18324135101511909607,"high":14684207395166810797}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489366495161243379,"id":"test-1","namespace":"default","sink_uri":"mysql://root@127.0.0.1:3306","create_time":"2025-04-04T16:07:19.839160467+08:00","start_ts":457114659012739074,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114659012739074,"checkpoint_ts":457114659012739074,"checkpoint_time":"2025-04-04 16:07:19.813","gid":{"low":16232394410305025397,"high":11066811453105502100}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... table partition_table.finish_mark not exists for 9-th check, retry later check diff failed 2-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.7595.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457114658895560705 PASS coverage: 1.9% of statements in github.com/pingcap/ticdc/...' + echo 457114658895560705 PASS coverage: 1.9% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:20 CST 2025] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates ']' + GO_FAILPOINTS= + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.30273029.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.5782.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:4000 -c test-2 --server http://127.0.0.1:8400 --config=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/bdr_mode/conf/down.toml Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: test-2 Info: {"upstream_id":7489366492174113348,"id":"test-2","namespace":"default","sink_uri":"mysql://root@127.0.0.1:4000","create_time":"2025-04-04T16:07:21.555708197+08:00","start_ts":457114659457335300,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114659457335300,"checkpoint_ts":457114659457335300,"checkpoint_time":"2025-04-04 16:07:21.509","gid":{"low":8524504835861039914,"high":11966798654767023266}} PASS coverage: 2.8% of statements in github.com/pingcap/ticdc/... + set +x + tso='457114659152461826 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114659152461826 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:21 CST 2025] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.76397641.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 10-th check, retry later check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:21 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf4ca025-4416-4077-aa61-dba033be8a8e {"id":"bf4ca025-4416-4077-aa61-dba033be8a8e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754039} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf4ca025-4416-4077-aa61-dba033be8a8e {"id":"bf4ca025-4416-4077-aa61-dba033be8a8e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754039} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf4ca025-4416-4077-aa61-dba033be8a8e {"id":"bf4ca025-4416-4077-aa61-dba033be8a8e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754039} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114658422652929 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.5666.out cli changefeed create --start-ts=457114658422652929 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489366517910410119,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:22.351134378+08:00","start_ts":457114658422652929,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114658422652929,"checkpoint_ts":457114658422652929,"checkpoint_time":"2025-04-04 16:07:17.562","gid":{"low":5350837396807433449,"high":14629577223381727151}} PASS + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * ALPN, offering h2 * ALPN, offering http/1.1 * successfully set certificate verify locations: * CAfile: /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem CApath: none } [5 bytes data] * TLSv1.3 (OUT), TLS handshake, Client hello (1): } [512 bytes data] * TLSv1.3 (IN), TLS handshake, Server hello (2): { [122 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): { [6 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Request CERT (13): { [142 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Certificate (11): { [928 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, CERT verify (15): { [264 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Finished (20): { [36 bytes data] * TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Certificate (11): } [1814 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, CERT verify (15): } [264 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Finished (20): } [36 bytes data] * SSL connection using TLSv1.3 / TLS_AES_128_GCM_SHA256 * ALPN, server did not agree to a protocol * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * subjectAltName: host "127.0.0.1" matched cert's IP address! * issuer: C=CN; ST=Beijing; L=Beijing; O=PingCAP; CN=My own CA * SSL certificate verify ok. * Server auth using Basic with user 'ticdc' } [5 bytes data] * TLSv1.3 (OUT), TLS app data, [no content] (0): } [1 bytes data] > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > { [5 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): { [2834 bytes data] * TLSv1.3 (IN), TLS app data, [no content] (0): { [1 bytes data] < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:22 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efe792a1-a644-4e88-b46f-b545e2ffb03c {"id":"efe792a1-a644-4e88-b46f-b545e2ffb03c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754039} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efe792a1-a644-4e88-b46f-b545e2ffb03c {"id":"efe792a1-a644-4e88-b46f-b545e2ffb03c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754039} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efe792a1-a644-4e88-b46f-b545e2ffb03c {"id":"efe792a1-a644-4e88-b46f-b545e2ffb03c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754039} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c' + grep -q 'etcd info' + break + set +x coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x table resourcecontrol.finish_mark not exists for 1-th check, retry later + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * ALPN, offering h2 * ALPN, offering http/1.1 * successfully set certificate verify locations: * CAfile: /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem CApath: none } [5 bytes data] * TLSv1.3 (OUT), TLS handshake, Client hello (1): } [512 bytes data] * TLSv1.3 (IN), TLS handshake, Server hello (2): { [122 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): { [6 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Request CERT (13): { [142 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Certificate (11): { [928 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, CERT verify (15): { [264 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Finished (20): { [36 bytes data] * TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Certificate (11): } [1814 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, CERT verify (15): } [264 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Finished (20): } [36 bytes data] * SSL connection using TLSv1.3 / TLS_AES_128_GCM_SHA256 * ALPN, server did not agree to a protocol * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * subjectAltName: host "127.0.0.1" matched cert's IP address! * issuer: C=CN; ST=Beijing; L=Beijing; O=PingCAP; CN=My own CA * SSL certificate verify ok. * Server auth using Basic with user 'ticdc' } [5 bytes data] * TLSv1.3 (OUT), TLS app data, [no content] (0): } [1 bytes data] > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > { [5 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): { [2834 bytes data] * TLSv1.3 (IN), TLS app data, [no content] (0): { [1 bytes data] < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:23 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3739517f-4ba7-46a8-a72a-3472a983bf0c {"id":"3739517f-4ba7-46a8-a72a-3472a983bf0c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754041} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f63ff3 3739517f-4ba7-46a8-a72a-3472a983bf0c /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f63ff3 3739517f-4ba7-46a8-a72a-3472a983bf0c' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3739517f-4ba7-46a8-a72a-3472a983bf0c {"id":"3739517f-4ba7-46a8-a72a-3472a983bf0c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754041} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f63ff3 3739517f-4ba7-46a8-a72a-3472a983bf0c /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f63ff3 3739517f-4ba7-46a8-a72a-3472a983bf0c' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3739517f-4ba7-46a8-a72a-3472a983bf0c {"id":"3739517f-4ba7-46a8-a72a-3472a983bf0c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754041} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f63ff3 3739517f-4ba7-46a8-a72a-3472a983bf0c /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f63ff3 3739517f-4ba7-46a8-a72a-3472a983bf0c' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:07:24 CST 2025] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.56225624.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/mysql_sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table partition_table.finish_mark exists check diff successfully check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/partition_table/stdout.log no DATA RACE found [Fri Apr 4 16:07:24 CST 2025] <<<<<< run test case partition_table success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:24 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d4edd01-fd77-4365-a757-d4ba7e12eb05 {"id":"0d4edd01-fd77-4365-a757-d4ba7e12eb05","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754042} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6e275eb 0d4edd01-fd77-4365-a757-d4ba7e12eb05 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6e275eb 0d4edd01-fd77-4365-a757-d4ba7e12eb05' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d4edd01-fd77-4365-a757-d4ba7e12eb05 {"id":"0d4edd01-fd77-4365-a757-d4ba7e12eb05","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754042} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6e275eb 0d4edd01-fd77-4365-a757-d4ba7e12eb05 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6e275eb 0d4edd01-fd77-4365-a757-d4ba7e12eb05' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d4edd01-fd77-4365-a757-d4ba7e12eb05 {"id":"0d4edd01-fd77-4365-a757-d4ba7e12eb05","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754042} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6e275eb 0d4edd01-fd77-4365-a757-d4ba7e12eb05 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6e275eb 0d4edd01-fd77-4365-a757-d4ba7e12eb05' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.7712.out cli changefeed create --start-ts=457114659152461826 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 1558632552218024995412248656671160726933 Info: {"upstream_id":7489366527657414658,"id":"1558632552218024995412248656671160726933","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:25.146263034+08:00","start_ts":457114659152461826,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114659152461826,"checkpoint_ts":457114659152461826,"checkpoint_time":"2025-04-04 16:07:20.346","gid":{"low":15586325522180249954,"high":12248656671160726933}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3110.out cli changefeed create --start-ts=457114658895560705 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name table resourcecontrol.finish_mark exists check diff successfully [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489366544682273763,"id":"custom-changefeed-name","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:26.354237589+08:00","start_ts":457114658895560705,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114658895560705,"checkpoint_ts":457114658895560705,"checkpoint_time":"2025-04-04 16:07:19.366","gid":{"low":9459547666138716088,"high":14452482198457402269}} PASS cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/resourcecontrol/stdout.log no DATA RACE found [Fri Apr 4 16:07:26 CST 2025] <<<<<< run test case resourcecontrol success! >>>>>> table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5615) - No such process wait process 5615 exit for 1-th time... process 5615 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:07:26 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.57555757.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x table savepoint.finish_mark not exists for 1-th check, retry later check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * ALPN, offering h2 * ALPN, offering http/1.1 * successfully set certificate verify locations: * CAfile: /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem CApath: none } [5 bytes data] * TLSv1.3 (OUT), TLS handshake, Client hello (1): } [512 bytes data] * TLSv1.3 (IN), TLS handshake, Server hello (2): { [122 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): { [6 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Request CERT (13): { [142 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Certificate (11): { [928 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, CERT verify (15): { [264 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Finished (20): { [36 bytes data] * TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Certificate (11): } [1814 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, CERT verify (15): } [264 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Finished (20): } [36 bytes data] * SSL connection using TLSv1.3 / TLS_AES_128_GCM_SHA256 * ALPN, server did not agree to a protocol * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * subjectAltName: host "127.0.0.1" matched cert's IP address! * issuer: C=CN; ST=Beijing; L=Beijing; O=PingCAP; CN=My own CA * SSL certificate verify ok. * Server auth using Basic with user 'ticdc' } [5 bytes data] * TLSv1.3 (OUT), TLS app data, [no content] (0): } [1 bytes data] > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > { [5 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): { [2834 bytes data] * TLSv1.3 (IN), TLS app data, [no content] (0): { [1 bytes data] < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:27 GMT < Content-Length: 1184 < Content-Type: text/plain; charset=utf-8 < { [1067 bytes data] * TLSv1.3 (IN), TLS app data, [no content] (0): { [1 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bac43f0-adf5-4975-8a53-b6c8b2e93252 {"id":"3bac43f0-adf5-4975-8a53-b6c8b2e93252","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754044} /tidb/cdc/default/__cdc_meta__/capture/efe792a1-a644-4e88-b46f-b545e2ffb03c {"id":"efe792a1-a644-4e88-b46f-b545e2ffb03c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754039} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f49465 3bac43f0-adf5-4975-8a53-b6c8b2e93252 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f49465 3bac43f0-adf5-4975-8a53-b6c8b2e93252' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bac43f0-adf5-4975-8a53-b6c8b2e93252 {"id":"3bac43f0-adf5-4975-8a53-b6c8b2e93252","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754044} /tidb/cdc/default/__cdc_meta__/capture/efe792a1-a644-4e88-b46f-b545e2ffb03c {"id":"efe792a1-a644-4e88-b46f-b545e2ffb03c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754039} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f49465 3bac43f0-adf5-4975-8a53-b6c8b2e93252 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f49465 3bac43f0-adf5-4975-8a53-b6c8b2e93252' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bac43f0-adf5-4975-8a53-b6c8b2e93252 {"id":"3bac43f0-adf5-4975-8a53-b6c8b2e93252","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754044} /tidb/cdc/default/__cdc_meta__/capture/efe792a1-a644-4e88-b46f-b545e2ffb03c {"id":"efe792a1-a644-4e88-b46f-b545e2ffb03c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754039} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f49465 3bac43f0-adf5-4975-8a53-b6c8b2e93252 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f493dc efe792a1-a644-4e88-b46f-b545e2ffb03c /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f49465 3bac43f0-adf5-4975-8a53-b6c8b2e93252' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/mysql_sink_retry Starting Upstream PD... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366578265175520 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5cbee00001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:3666, start at 2025-04-04 16:07:27.416623158 +0800 CST m=+3.741492826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:27.428 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:27.416 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:27.416 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366578265175520 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5cbee00001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:3666, start at 2025-04-04 16:07:27.416623158 +0800 CST m=+3.741492826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:27.428 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:27.416 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:27.416 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366574107313973 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5cc2000006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:3739, start at 2025-04-04 16:07:27.627690739 +0800 CST m=+3.792702810 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:27.639 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:27.616 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:27.616 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x table test.simple exists table test.`simple-dash` exists + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c custom-changefeed-name -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114658895560705, "checkpoint_time": "2025-04-04 16:07:19.366", "error": null }' + echo '{ "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114658895560705, "checkpoint_time": "2025-04-04 16:07:19.366", "error": null }' { "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114658895560705, "checkpoint_time": "2025-04-04 16:07:19.366", "error": null } ++ echo '{' '"upstream_id":' 7489366544682273763, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114658895560705, '"checkpoint_time":' '"2025-04-04' '16:07:19.366",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366544682273763, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114658895560705, '"checkpoint_time":' '"2025-04-04' '16:07:19.366",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: https://127.0.0.1:2579 Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3280.out cli changefeed --changefeed-id custom-changefeed-name pause PASS coverage: 2.1% of statements in github.com/pingcap/ticdc/... table savepoint.finish_mark exists check diff failed 1-th time, retry later pass test: check health pass test: get status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:29 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41388f48-95a4-458a-b326-6aa235a259fb {"id":"41388f48-95a4-458a-b326-6aa235a259fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754046} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5350837396807433449,"high":14629577223381727151},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366517910410119,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:22.351134378+08:00","start-ts":457114658422652929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114659667836938} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114660061052949,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41388f48-95a4-458a-b326-6aa235a259fb {"id":"41388f48-95a4-458a-b326-6aa235a259fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754046} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5350837396807433449,"high":14629577223381727151},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366517910410119,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:22.351134378+08:00","start-ts":457114658422652929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114659667836938} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114660061052949,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41388f48-95a4-458a-b326-6aa235a259fb {"id":"41388f48-95a4-458a-b326-6aa235a259fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754046} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5350837396807433449,"high":14629577223381727151},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366517910410119,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:22.351134378+08:00","start-ts":457114658422652929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114659667836938} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114660061052949,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:07:29 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.58345836.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... check diff successfully pass test: create changefeed command: check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c changefeed-test1 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366544095705795, "id": "changefeed-test1", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661562351632, "checkpoint_time": "2025-04-04 16:07:29.539", "error": null }' + echo '{ "upstream_id": 7489366544095705795, "id": "changefeed-test1", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661562351632, "checkpoint_time": "2025-04-04 16:07:29.539", "error": null }' { "upstream_id": 7489366544095705795, "id": "changefeed-test1", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661562351632, "checkpoint_time": "2025-04-04 16:07:29.539", "error": null } ++ echo '{' '"upstream_id":' 7489366544095705795, '"id":' '"changefeed-test1",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661562351632, '"checkpoint_time":' '"2025-04-04' '16:07:29.539",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366544095705795, '"id":' '"changefeed-test1",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661562351632, '"checkpoint_time":' '"2025-04-04' '16:07:29.539",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully command: check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c changefeed-test2 -s ++ grep -v 'Command to ticdc' TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... + info='{ "upstream_id": 7489366544095705795, "id": "changefeed-test2", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661588566021, "checkpoint_time": "2025-04-04 16:07:29.639", "error": null }' + echo '{ "upstream_id": 7489366544095705795, "id": "changefeed-test2", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661588566021, "checkpoint_time": "2025-04-04 16:07:29.639", "error": null }' { "upstream_id": 7489366544095705795, "id": "changefeed-test2", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661588566021, "checkpoint_time": "2025-04-04 16:07:29.639", "error": null } ++ echo '{' '"upstream_id":' 7489366544095705795, '"id":' '"changefeed-test2",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661588566021, '"checkpoint_time":' '"2025-04-04' '16:07:29.639",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366544095705795, '"id":' '"changefeed-test2",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661588566021, '"checkpoint_time":' '"2025-04-04' '16:07:29.639",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully command: check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c changefeed-test3 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366544095705795, "id": "changefeed-test3", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661601673223, "checkpoint_time": "2025-04-04 16:07:29.689", "error": null }' + echo '{ "upstream_id": 7489366544095705795, "id": "changefeed-test3", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661601673223, "checkpoint_time": "2025-04-04 16:07:29.689", "error": null }' { "upstream_id": 7489366544095705795, "id": "changefeed-test3", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661601673223, "checkpoint_time": "2025-04-04 16:07:29.689", "error": null } ++ echo '{' '"upstream_id":' 7489366544095705795, '"id":' '"changefeed-test3",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661601673223, '"checkpoint_time":' '"2025-04-04' '16:07:29.689",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366544095705795, '"id":' '"changefeed-test3",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661601673223, '"checkpoint_time":' '"2025-04-04' '16:07:29.689",' '"error":' null '}' ++ jq -r .error.message + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.5025.out cli tso query --pd=http://127.0.0.1:2379 + set +x + message=null + [[ ! null =~ null ]] run task successfully command: check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c changefeed-test4 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366544095705795, "id": "changefeed-test4", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661614518280, "checkpoint_time": "2025-04-04 16:07:29.738", "error": null }' + echo '{ "upstream_id": 7489366544095705795, "id": "changefeed-test4", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661614518280, "checkpoint_time": "2025-04-04 16:07:29.738", "error": null }' { "upstream_id": 7489366544095705795, "id": "changefeed-test4", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661614518280, "checkpoint_time": "2025-04-04 16:07:29.738", "error": null } ++ echo '{' '"upstream_id":' 7489366544095705795, '"id":' '"changefeed-test4",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661614518280, '"checkpoint_time":' '"2025-04-04' '16:07:29.738",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366544095705795, '"id":' '"changefeed-test4",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661614518280, '"checkpoint_time":' '"2025-04-04' '16:07:29.738",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 2-th time... table row_format.finish_mark not exists for 1-th check, retry later check diff successfully wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/savepoint/stdout.log no DATA RACE found [Fri Apr 4 16:07:31 CST 2025] <<<<<< run test case savepoint success! >>>>>> wait process cdc.test exit for 4-th time... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114661866962945 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114661866962945 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:32 CST 2025] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.50535055.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 5-th time... table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c test-normal -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366498814786281, "id": "test-normal", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661399035910, "checkpoint_time": "2025-04-04 16:07:28.916", "error": null }' + echo '{ "upstream_id": 7489366498814786281, "id": "test-normal", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661399035910, "checkpoint_time": "2025-04-04 16:07:28.916", "error": null }' { "upstream_id": 7489366498814786281, "id": "test-normal", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661399035910, "checkpoint_time": "2025-04-04 16:07:28.916", "error": null } ++ echo '{' '"upstream_id":' 7489366498814786281, '"id":' '"test-normal",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661399035910, '"checkpoint_time":' '"2025-04-04' '16:07:28.916",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366498814786281, '"id":' '"test-normal",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661399035910, '"checkpoint_time":' '"2025-04-04' '16:07:28.916",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c test-error-1 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366498814786281, "id": "test-error-1", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661661179913, "checkpoint_time": "2025-04-04 16:07:29.916", "error": null }' + echo '{ "upstream_id": 7489366498814786281, "id": "test-error-1", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661661179913, "checkpoint_time": "2025-04-04 16:07:29.916", "error": null }' { "upstream_id": 7489366498814786281, "id": "test-error-1", "namespace": "default", "state": "normal", "checkpoint_tso": 457114661661179913, "checkpoint_time": "2025-04-04 16:07:29.916", "error": null } ++ echo '{' '"upstream_id":' 7489366498814786281, '"id":' '"test-error-1",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661661179913, '"checkpoint_time":' '"2025-04-04' '16:07:29.916",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366498814786281, '"id":' '"test-error-1",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114661661179913, '"checkpoint_time":' '"2025-04-04' '16:07:29.916",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] Attempt 1 of 10: check_changefeed_state http://127.0.0.1:2379 test-error-2 failed ErrSyncRenameTableFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir=ErrSyncRenameTableFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c test-error-2 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366498814786281, "id": "test-error-2", "namespace": "default", "state": "failed", "checkpoint_tso": 457114661136891913, "checkpoint_time": "2025-04-04 16:07:27.916", "error": { "time": "2025-04-04T16:07:31.260576981+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrChangefeedUnretryable", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''138'\'', ddl query: [RENAME TABLE `multi_tables_ddl_test`.`t11` TO `multi_tables_ddl_test`.`t9`], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7489366498814786281, "id": "test-error-2", "namespace": "default", "state": "failed", "checkpoint_tso": 457114661136891913, "checkpoint_time": "2025-04-04 16:07:27.916", "error": { "time": "2025-04-04T16:07:31.260576981+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrChangefeedUnretryable", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''138'\'', ddl query: [RENAME TABLE `multi_tables_ddl_test`.`t11` TO `multi_tables_ddl_test`.`t9`], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7489366498814786281, "id": "test-error-2", "namespace": "default", "state": "failed", "checkpoint_tso": 457114661136891913, "checkpoint_time": "2025-04-04 16:07:27.916", "error": { "time": "2025-04-04T16:07:31.260576981+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrChangefeedUnretryable", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '138', ddl query: [RENAME TABLE `multi_tables_ddl_test`.`t11` TO `multi_tables_ddl_test`.`t9`], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ echo '{' '"upstream_id":' 7489366498814786281, '"id":' '"test-error-2",' '"namespace":' '"default",' '"state":' '"failed",' '"checkpoint_tso":' 457114661136891913, '"checkpoint_time":' '"2025-04-04' '16:07:27.916",' '"error":' '{' '"time":' '"2025-04-04T16:07:31.260576981+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrChangefeedUnretryable",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''138'\'',' ddl query: '[RENAME' TABLE '`multi_tables_ddl_test`.`t11`' TO '`multi_tables_ddl_test`.`t9`],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7489366498814786281, '"id":' '"test-error-2",' '"namespace":' '"default",' '"state":' '"failed",' '"checkpoint_tso":' 457114661136891913, '"checkpoint_time":' '"2025-04-04' '16:07:27.916",' '"error":' '{' '"time":' '"2025-04-04T16:07:31.260576981+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrChangefeedUnretryable",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''138'\'',' ddl query: '[RENAME' TABLE '`multi_tables_ddl_test`.`t11`' TO '`multi_tables_ddl_test`.`t9`],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''138'\'', ddl query: [RENAME TABLE `multi_tables_ddl_test`.`t11` TO `multi_tables_ddl_test`.`t9`], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '138', ddl query: [RENAME TABLE `multi_tables_ddl_test`.`t11` TO `multi_tables_ddl_test`.`t9`], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] Command succeeded on attempt 1 check diff successfully wait process cdc.test exit for 6-th time... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/multi_tables_ddl/stdout.log no DATA RACE found [Fri Apr 4 16:07:32 CST 2025] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41388f48-95a4-458a-b326-6aa235a259fb {"id":"41388f48-95a4-458a-b326-6aa235a259fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754046} /tidb/cdc/default/__cdc_meta__/capture/a9727a03-ef64-41f9-84e2-c17dadf4d077 {"id":"a9727a03-ef64-41f9-84e2-c17dadf4d077","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754049} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5350837396807433449,"high":14629577223381727151},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366517910410119,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:22.351134378+08:00","start-ts":457114658422652929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114659667836938} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114660061052949,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41388f48-95a4-458a-b326-6aa235a259fb {"id":"41388f48-95a4-458a-b326-6aa235a259fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754046} /tidb/cdc/default/__cdc_meta__/capture/a9727a03-ef64-41f9-84e2-c17dadf4d077 {"id":"a9727a03-ef64-41f9-84e2-c17dadf4d077","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754049} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5350837396807433449,"high":14629577223381727151},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366517910410119,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:22.351134378+08:00","start-ts":457114658422652929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114659667836938} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114660061052949,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41388f48-95a4-458a-b326-6aa235a259fb {"id":"41388f48-95a4-458a-b326-6aa235a259fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754046} /tidb/cdc/default/__cdc_meta__/capture/a9727a03-ef64-41f9-84e2-c17dadf4d077 {"id":"a9727a03-ef64-41f9-84e2-c17dadf4d077","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754049} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc16e9 bf4ca025-4416-4077-aa61-dba033be8a8e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc17c8 41388f48-95a4-458a-b326-6aa235a259fb /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5350837396807433449,"high":14629577223381727151},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366517910410119,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:22.351134378+08:00","start-ts":457114658422652929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114659667836938} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114660061052949,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.5909.out cli capture list wait process cdc.test exit for 7-th time... table row_format.finish_mark exists check diff successfully check diff failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/row_format/stdout.log no DATA RACE found [Fri Apr 4 16:07:33 CST 2025] <<<<<< run test case row_format success! >>>>>> wait process cdc.test exit for 8-th time... + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c custom-changefeed-name -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114659262562316, "checkpoint_time": "2025-04-04 16:07:20.766", "error": null }' + echo '{ "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114659262562316, "checkpoint_time": "2025-04-04 16:07:20.766", "error": null }' { "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114659262562316, "checkpoint_time": "2025-04-04 16:07:20.766", "error": null } ++ echo '{' '"upstream_id":' 7489366544682273763, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"stopped",' '"checkpoint_tso":' 457114659262562316, '"checkpoint_time":' '"2025-04-04' '16:07:20.766",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489366544682273763, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"stopped",' '"checkpoint_tso":' 457114659262562316, '"checkpoint_time":' '"2025-04-04' '16:07:20.766",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3350.out cli changefeed update --pd=https://127.0.0.1:2579 --config=/tmp/tidb_cdc_test/cli_tls_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7489366544682273763,"id":"custom-changefeed-name","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:26.354237589+08:00","start_ts":457114658895560705,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":true,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114659262562316,"checkpoint_ts":457114659262562316,"checkpoint_time":"2025-04-04 16:07:20.766","gid":{"low":9459547666138716088,"high":14452482198457402269}} PASS coverage: 3.2% of statements in github.com/pingcap/ticdc/... wait process cdc.test exit for 9-th time... + set +x + ans='[ { "id": "41388f48-95a4-458a-b326-6aa235a259fb", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "a9727a03-ef64-41f9-84e2-c17dadf4d077", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"41388f48-95a4-458a-b326-6aa235a259fb",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"a9727a03-ef64-41f9-84e2-c17dadf4d077",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=a9727a03-ef64-41f9-84e2-c17dadf4d077 + '[' -z a9727a03-ef64-41f9-84e2-c17dadf4d077 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d a9727a03-ef64-41f9-84e2-c17dadf4d077 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.5936.out cli changefeed move-table -c test -t 114 -d a9727a03-ef64-41f9-84e2-c17dadf4d077 table test.verify_table_eligible exists pass test: list changefeed ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_F/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 10-th time... pass test: get changefeed Verifying downstream PD is started... Starting Upstream TiKV... wait process cdc.test exit for 11-th time... check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:35 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1df79b5-5544-46f1-9719-95300dd27395 {"id":"f1df79b5-5544-46f1-9719-95300dd27395","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754052} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd70928e3 f1df79b5-5544-46f1-9719-95300dd27395 /tidb/cdc/default/__cdc_meta__/owner/223195ffd70928e3 f1df79b5-5544-46f1-9719-95300dd27395' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1df79b5-5544-46f1-9719-95300dd27395 {"id":"f1df79b5-5544-46f1-9719-95300dd27395","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754052} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd70928e3 f1df79b5-5544-46f1-9719-95300dd27395 /tidb/cdc/default/__cdc_meta__/owner/223195ffd70928e3 f1df79b5-5544-46f1-9719-95300dd27395' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1df79b5-5544-46f1-9719-95300dd27395 {"id":"f1df79b5-5544-46f1-9719-95300dd27395","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754052} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd70928e3 f1df79b5-5544-46f1-9719-95300dd27395 /tidb/cdc/default/__cdc_meta__/owner/223195ffd70928e3 f1df79b5-5544-46f1-9719-95300dd27395' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114661866962945 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.5117.out cli changefeed create --start-ts=457114661866962945 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 115496635543970691646973138392175238544 Info: {"upstream_id":7489366578265175520,"id":"115496635543970691646973138392175238544","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:35.487167805+08:00","start_ts":457114661866962945,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114661866962945,"checkpoint_ts":457114661866962945,"checkpoint_time":"2025-04-04 16:07:30.701","gid":{"low":11549663554397069164,"high":6973138392175238544}} PASS TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3390.out cli changefeed --changefeed-id custom-changefeed-name resume coverage: 2.7% of statements in github.com/pingcap/ticdc/... PASS coverage: 2.3% of statements in github.com/pingcap/ticdc/... wait process cdc.test exit for 12-th time... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/split_region/stdout.log no DATA RACE found [Fri Apr 4 16:07:35 CST 2025] <<<<<< run test case split_region success! >>>>>> wait process cdc.test exit for 13-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_E/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366609567989256 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5d3bbc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:6465, start at 2025-04-04 16:07:35.410240513 +0800 CST m=+3.452025279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:35.419 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:35.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:35.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366609567989256 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5d3bbc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:6465, start at 2025-04-04 16:07:35.410240513 +0800 CST m=+3.452025279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:35.419 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:35.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:35.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366610086361627 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5d3bb4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:6540, start at 2025-04-04 16:07:35.429689208 +0800 CST m=+3.306351846 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:35.437 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:35.455 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:35.455 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/mysql_sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/mysql_sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/mysql_sink_retry/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/mysql_sink_retry/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/mysql_sink_retry/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } pass test: pause changefeed wait process cdc.test exit for 14-th time... + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 + set +x Command succeeded on attempt 1 command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x cdc.test: no process found wait process cdc.test exit for 15-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/kv_client_stream_reconnect/stdout.log no DATA RACE found [Fri Apr 4 16:07:37 CST 2025] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> table ddl_sequence.finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: update changefeed start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_F Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mysql_sink_retry.cli.7746.out cli tso query --pd=http://127.0.0.1:2379 pass test: resume changefeed Run 5 test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5760) - No such process wait process 5760 exit for 1-th time... process 5760 already exit table ddl_sequence.finish_mark exists check diff failed 1-th time, retry later check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: move table + set +x + tso='457114664018116609 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114664018116609 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "operationcount"="0" "mysql.host"="127.0.0.1" "readproportion"="0" "dotransactions"="false" "mysql.user"="root" "insertproportion"="0" "workload"="core" "requestdistribution"="uniform" "mysql.port"="4000" "scanproportion"="0" "updateproportion"="0" "mysql.db"="sink_retry" "recordcount"="10" "readallfields"="true" "threadcount"="2" ********************************************** Run finished, takes 9.274854ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1703.6, Avg(us): 1726, Min(us): 990, Max(us): 4403, 95th(us): 5000, 99th(us): 5000 [Fri Apr 4 16:07:40 CST 2025] <<<<<< START cdc server in mysql_sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/sink/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mysql_sink_retry.77987800.out server --log-file /tmp/tidb_cdc_test/mysql_sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/mysql_sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c custom-changefeed-name -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114663758331906, "checkpoint_time": "2025-04-04 16:07:37.916", "error": null }' + echo '{ "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114663758331906, "checkpoint_time": "2025-04-04 16:07:37.916", "error": null }' { "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114663758331906, "checkpoint_time": "2025-04-04 16:07:37.916", "error": null } ++ echo '{' '"upstream_id":' 7489366544682273763, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114663758331906, '"checkpoint_time":' '"2025-04-04' '16:07:37.916",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366544682273763, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114663758331906, '"checkpoint_time":' '"2025-04-04' '16:07:37.916",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3470.out cli changefeed --changefeed-id custom-changefeed-name remove start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_E Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 457114664020475906 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... pass test: set log level Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_C/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/ddl_sequence/stdout.log no DATA RACE found [Fri Apr 4 16:07:42 CST 2025] <<<<<< run test case ddl_sequence success! >>>>>> + set +x pass test: remove changefeed check diff failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:43 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf272b55-bcc6-444a-afa1-ff8d8cab00e2 {"id":"bf272b55-bcc6-444a-afa1-ff8d8cab00e2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754060} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7313af4 bf272b55-bcc6-444a-afa1-ff8d8cab00e2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7313af4 bf272b55-bcc6-444a-afa1-ff8d8cab00e2' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf272b55-bcc6-444a-afa1-ff8d8cab00e2 {"id":"bf272b55-bcc6-444a-afa1-ff8d8cab00e2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754060} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7313af4 bf272b55-bcc6-444a-afa1-ff8d8cab00e2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7313af4 bf272b55-bcc6-444a-afa1-ff8d8cab00e2' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf272b55-bcc6-444a-afa1-ff8d8cab00e2 {"id":"bf272b55-bcc6-444a-afa1-ff8d8cab00e2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754060} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7313af4 bf272b55-bcc6-444a-afa1-ff8d8cab00e2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7313af4 bf272b55-bcc6-444a-afa1-ff8d8cab00e2' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mysql_sink_retry.cli.7859.out cli changefeed create --start-ts=457114664018116609 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 787299556266361802214174187522496598929 Info: {"upstream_id":7489366609567989256,"id":"787299556266361802214174187522496598929","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T16:07:43.738446935+08:00","start_ts":457114664018116609,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114664018116609,"checkpoint_ts":457114664018116609,"checkpoint_time":"2025-04-04 16:07:38.907","gid":{"low":7872995562663618022,"high":14174187522496598929}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... pass test: resign owner VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366634217063346 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5da1e0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:3554, start at 2025-04-04 16:07:41.964124131 +0800 CST m=+4.378516501 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:41.974 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:42.044 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:42.044 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366634217063346 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5da1e0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:3554, start at 2025-04-04 16:07:41.964124131 +0800 CST m=+4.378516501 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:41.974 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:42.044 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:42.044 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366630399698272 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5da6d40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:3624, start at 2025-04-04 16:07:42.266264396 +0800 CST m=+4.495260799 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:42.275 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:42.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:42.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_C Starting Upstream PD... pass test: get tso Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x changefeed count 0 check pass, pd_addr: https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3540.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489366544682273763,"id":"custom-changefeed-name","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:45.711058313+08:00","start_ts":457114665789947907,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114665789947907,"checkpoint_ts":457114665789947907,"checkpoint_time":"2025-04-04 16:07:45.666","gid":{"low":11836465163790157899,"high":12909107374899450769}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/changefeed_resume_with_checkpoint_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... pass test: verify table [Fri Apr 4 16:07:46 CST 2025] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.48924894.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff successfully pass test: unsafe apis + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_D/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366663257672681 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5decb00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:8683, start at 2025-04-04 16:07:46.736670002 +0800 CST m=+3.345303415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:46.744 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:46.732 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:46.732 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366663257672681 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5decb00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:8683, start at 2025-04-04 16:07:46.736670002 +0800 CST m=+3.345303415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:46.744 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:46.732 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:46.732 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366660793376449 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5df01c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:8757, start at 2025-04-04 16:07:46.98616207 +0800 CST m=+3.423039187 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:47.039 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:47.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:47.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366663460751938 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5df6d8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:8854, start at 2025-04-04 16:07:47.399502674 +0800 CST m=+3.716184218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:47.483 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:47.482 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:47.482 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366663460751938 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5df6d8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:8854, start at 2025-04-04 16:07:47.399502674 +0800 CST m=+3.716184218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:47.483 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:47.482 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:47.482 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366664780313280 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5df9f80001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:8922, start at 2025-04-04 16:07:47.58289115 +0800 CST m=+3.694860114 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:47.593 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:47.582 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:47.582 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/http_api_tls/stdout_http_api_tls_tls1.log /tmp/tidb_cdc_test/http_api_tls/stdout_http_api_tls_tls2.log no DATA RACE found [Fri Apr 4 16:07:48 CST 2025] <<<<<< run test case http_api_tls success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:49 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30144618-01ae-4c33-97c4-fbbe3a1eae40 {"id":"30144618-01ae-4c33-97c4-fbbe3a1eae40","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754066} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd744f9ea 30144618-01ae-4c33-97c4-fbbe3a1eae40 /tidb/cdc/default/__cdc_meta__/owner/223195ffd744f9ea 30144618-01ae-4c33-97c4-fbbe3a1eae40' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30144618-01ae-4c33-97c4-fbbe3a1eae40 {"id":"30144618-01ae-4c33-97c4-fbbe3a1eae40","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754066} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd744f9ea 30144618-01ae-4c33-97c4-fbbe3a1eae40 /tidb/cdc/default/__cdc_meta__/owner/223195ffd744f9ea 30144618-01ae-4c33-97c4-fbbe3a1eae40' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30144618-01ae-4c33-97c4-fbbe3a1eae40 {"id":"30144618-01ae-4c33-97c4-fbbe3a1eae40","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754066} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd744f9ea 30144618-01ae-4c33-97c4-fbbe3a1eae40 /tidb/cdc/default/__cdc_meta__/owner/223195ffd744f9ea 30144618-01ae-4c33-97c4-fbbe3a1eae40' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.4952.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/batch_add_table/conf/changefeed.toml Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: 1061174350790855993014211106956093293186 Info: {"upstream_id":7489366634217063346,"id":"1061174350790855993014211106956093293186","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:49.577225777+08:00","start_ts":457114666798940166,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*","!test.t_1"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114666798940166,"checkpoint_ts":457114666798940166,"checkpoint_time":"2025-04-04 16:07:49.515","gid":{"low":10611743507908559930,"high":14211106956093293186}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.10037.out cli tso query --pd=http://127.0.0.1:2379 + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem -c custom-changefeed-name -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114666379771907, "checkpoint_time": "2025-04-04 16:07:47.916", "error": null }' + echo '{ "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114666379771907, "checkpoint_time": "2025-04-04 16:07:47.916", "error": null }' { "upstream_id": 7489366544682273763, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114666379771907, "checkpoint_time": "2025-04-04 16:07:47.916", "error": null } ++ echo '{' '"upstream_id":' 7489366544682273763, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114666379771907, '"checkpoint_time":' '"2025-04-04' '16:07:47.916",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366544682273763, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114666379771907, '"checkpoint_time":' '"2025-04-04' '16:07:47.916",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_D Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.10275.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3648.out cli unsafe delete-service-gc-safepoint + set +x Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table test.t_1 not exists for 1-th check, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... + set +x + tso='457114667019403266 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114667019403266 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:51 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1006510067.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + tso='457114667116658689 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114667116658689 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:52 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1030710309.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3672.out cli unsafe reset --no-confirm --pd=https://127.0.0.1:2579 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) reset and all metadata truncated in PD! PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t_1 exists table test.t_2 exists table test.t_3 exists table test.t_4 exists table test.t_5 exists table test.t_6 exists table test.t_7 exists table test.t_8 exists table test.t_9 exists table test.t_10 exists table test.t_11 exists table test.t_12 exists table test.t_13 exists table test.t_14 exists table test.t_15 exists table test.t_16 exists table test.t_17 exists table test.t_18 exists table test.t_19 exists table test.t_20 exists table test.t_21 exists table test.t_22 exists table test.t_23 exists table test.t_24 exists table test.t_25 exists table test.t_26 exists table test.t_27 exists table test.t_28 exists table test.t_29 exists table test.t_30 exists table test.t_31 exists table test.t_32 exists table test.t_33 exists table test.t_34 exists table test.t_35 exists table test.t_36 exists table test.t_37 exists table test.t_38 exists table test.t_39 exists table test.t_40 exists table test.t_41 exists table test.t_42 exists table test.t_43 exists table test.t_44 exists table test.t_45 exists table test.t_46 exists table test.t_47 exists table test.t_48 exists table test.t_49 exists table test.t_50 exists table test.t_51 exists table test.t_52 exists table test.t_53 exists table test.t_54 exists table test.t_55 exists table test.t_56 exists table test.t_57 exists table test.t_58 exists table test.t_59 exists table test.t_60 exists table test.t_61 exists table test.t_62 exists table test.t_63 exists table test.t_64 exists table test.t_65 exists table test.t_66 exists table test.t_67 exists table test.t_68 exists table test.t_69 exists table test.t_70 exists table test.t_71 exists table test.t_72 exists table test.t_73 exists table test.t_74 exists table test.t_75 exists table test.t_76 exists table test.t_77 exists table test.t_78 exists table test.t_79 exists table test.t_80 exists table test.t_81 exists table test.t_82 exists table test.t_83 exists table test.t_84 exists table test.t_85 exists table test.t_86 exists table test.t_87 exists + set +x command: ! ps -p 3032 > /dev/null 2>&1 run task failed 1-th time, retry later table test.t_88 exists table test.t_89 exists table test.t_90 exists table test.t_91 exists table test.t_92 exists table test.t_93 exists table test.t_94 exists table test.t_95 exists table test.t_96 exists table test.t_97 exists table test.t_98 exists Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release command: check_changefeed_state http://127.0.0.1:2379 53505158414159366234280295275672135858 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=53505158414159366234280295275672135858 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c 53505158414159366234280295275672135858 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366217850765779, "id": "53505158414159366234280295275672135858", "namespace": "default", "state": "finished", "checkpoint_tso": 457114664914911236, "checkpoint_time": "2025-04-04 16:07:42.328", "error": null }' + echo '{ "upstream_id": 7489366217850765779, "id": "53505158414159366234280295275672135858", "namespace": "default", "state": "finished", "checkpoint_tso": 457114664914911236, "checkpoint_time": "2025-04-04 16:07:42.328", "error": null }' { "upstream_id": 7489366217850765779, "id": "53505158414159366234280295275672135858", "namespace": "default", "state": "finished", "checkpoint_tso": 457114664914911236, "checkpoint_time": "2025-04-04 16:07:42.328", "error": null } ++ echo '{' '"upstream_id":' 7489366217850765779, '"id":' '"53505158414159366234280295275672135858",' '"namespace":' '"default",' '"state":' '"finished",' '"checkpoint_tso":' 457114664914911236, '"checkpoint_time":' '"2025-04-04' '16:07:42.328",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7489366217850765779, '"id":' '"53505158414159366234280295275672135858",' '"namespace":' '"default",' '"state":' '"finished",' '"checkpoint_tso":' 457114664914911236, '"checkpoint_time":' '"2025-04-04' '16:07:42.328",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/changefeed_finish/stdout.log no DATA RACE found [Fri Apr 4 16:07:48 CST 2025] <<<<<< run test case changefeed_finish success! >>>>>> table test.t_99 exists table test.t_100 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:54 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e69263c1-210d-4131-901b-5355d564527f {"id":"e69263c1-210d-4131-901b-5355d564527f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754072} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e69263c1-210d-4131-901b-5355d564527f {"id":"e69263c1-210d-4131-901b-5355d564527f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754072} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e69263c1-210d-4131-901b-5355d564527f {"id":"e69263c1-210d-4131-901b-5355d564527f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754072} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114667019403266 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.10129.out cli changefeed create --start-ts=457114667019403266 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489366663257672681,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:55.125103016+08:00","start_ts":457114667019403266,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114667019403266,"checkpoint_ts":457114667019403266,"checkpoint_time":"2025-04-04 16:07:50.356","gid":{"low":17100088867030923722,"high":15578063972551856770}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366693286195187 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5e63e40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:3967, start at 2025-04-04 16:07:54.368949264 +0800 CST m=+3.704402462 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:54.381 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:54.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:54.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366693286195187 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5e63e40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:3967, start at 2025-04-04 16:07:54.368949264 +0800 CST m=+3.704402462 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:54.381 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:54.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:54.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366690078714769 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5e6890000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:4041, start at 2025-04-04 16:07:54.691395135 +0800 CST m=+3.846122315 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:54.742 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:54.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:54.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:55 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 {"id":"c9d7f8b9-d783-41b8-80ac-cfecf6b026b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754072} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 {"id":"c9d7f8b9-d783-41b8-80ac-cfecf6b026b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754072} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 {"id":"c9d7f8b9-d783-41b8-80ac-cfecf6b026b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754072} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114667116658689 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.10367.out cli changefeed create --start-ts=457114667116658689 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: test Info: {"upstream_id":7489366663460751938,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:07:55.542651828+08:00","start_ts":457114667116658689,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114667116658689,"checkpoint_ts":457114667116658689,"checkpoint_time":"2025-04-04 16:07:50.727","gid":{"low":12919736653804788008,"high":4951989450191133113}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... table test.t_100 exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366685609419036 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5e6fe00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:8781, start at 2025-04-04 16:07:55.142874896 +0800 CST m=+3.981902759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:55.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:55.128 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:55.128 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366685609419036 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5e6fe00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:8781, start at 2025-04-04 16:07:55.142874896 +0800 CST m=+3.981902759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:55.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:55.128 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:55.128 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366690096841106 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5e71b4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:8842, start at 2025-04-04 16:07:55.275092517 +0800 CST m=+3.914598046 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:55.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:55.345 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:55.345 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/server_config_compatibility Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table batch_add_table.finish_mark not exists for 1-th check, retry later + set +x Command succeeded on attempt 1 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table fail_over_ddl_test.test2 not exists for 1-th check, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) command: ! ps -p 3032 > /dev/null 2>&1 run task successfully [Fri Apr 4 16:07:56 CST 2025] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.37053707.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1_restart.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1_restart --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366700864709933 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5e85080006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:3751, start at 2025-04-04 16:07:56.492143749 +0800 CST m=+3.579897621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:56.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:56.482 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:56.482 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366700864709933 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5e85080006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:3751, start at 2025-04-04 16:07:56.492143749 +0800 CST m=+3.579897621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:56.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:56.482 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:56.482 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366694629554007 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5e87f00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:3827, start at 2025-04-04 16:07:56.689706585 +0800 CST m=+3.601287675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:56.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:56.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:56.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.5243.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.10021.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_G/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table fail_over_ddl_test.test2 exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark exists table test.t_1 does not exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366707262181732 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ea2cc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:3777, start at 2025-04-04 16:07:58.403019668 +0800 CST m=+3.436390999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:58.410 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:58.465 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:58.465 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + set +x + tso='457114668947472385 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114668947472385 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:59 CST 2025] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.52815283.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10070) - No such process wait process 10070 exit for 1-th time... process 10070 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:07:59 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1019710199.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366707262181732 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ea2cc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:3777, start at 2025-04-04 16:07:58.403019668 +0800 CST m=+3.436390999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:58.410 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:58.465 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:58.465 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366710341514099 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ea2780005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:3852, start at 2025-04-04 16:07:58.37252179 +0800 CST m=+3.282448378 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:09:58.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:07:58.366 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:57:58.366 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * ALPN, offering h2 * ALPN, offering http/1.1 * successfully set certificate verify locations: * CAfile: /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_certificates/ca.pem CApath: none } [5 bytes data] * TLSv1.3 (OUT), TLS handshake, Client hello (1): } [512 bytes data] * TLSv1.3 (IN), TLS handshake, Server hello (2): { [122 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): { [6 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Request CERT (13): { [142 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Certificate (11): { [928 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, CERT verify (15): { [264 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Finished (20): { [36 bytes data] * TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Certificate (11): } [1814 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, CERT verify (15): } [264 bytes data] * TLSv1.3 (OUT), TLS handshake, [no content] (0): } [1 bytes data] * TLSv1.3 (OUT), TLS handshake, Finished (20): } [36 bytes data] * SSL connection using TLSv1.3 / TLS_AES_128_GCM_SHA256 * ALPN, server did not agree to a protocol * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * subjectAltName: host "127.0.0.1" matched cert's IP address! * issuer: C=CN; ST=Beijing; L=Beijing; O=PingCAP; CN=My own CA * SSL certificate verify ok. * Server auth using Basic with user 'ticdc' } [5 bytes data] * TLSv1.3 (OUT), TLS app data, [no content] (0): } [1 bytes data] > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > { [5 bytes data] * TLSv1.3 (IN), TLS handshake, [no content] (0): { [1 bytes data] * TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): { [2834 bytes data] * TLSv1.3 (IN), TLS app data, [no content] (0): { [1 bytes data] < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:07:59 GMT < Content-Length: 722 < Content-Type: text/plain; charset=utf-8 < { [722 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a685d056-6b50-4e29-9295-37ed48e4ded8 {"id":"a685d056-6b50-4e29-9295-37ed48e4ded8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754077} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f643cf a685d056-6b50-4e29-9295-37ed48e4ded8 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f643cf a685d056-6b50-4e29-9295-37ed48e4ded8 /tidb/cdc/default/default/changefeed/status/custom-changefeed-name {"checkpoint-ts":457114667952635905,"progress":1}' Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.5039.out cli tso query --pd=http://127.0.0.1:2379 table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10312) - No such process wait process 10312 exit for 1-th time... process 10312 already exit + set +x + tso='457114669083000838 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114669083000838 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:07:59 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1006310065.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a685d056-6b50-4e29-9295-37ed48e4ded8 {"id":"a685d056-6b50-4e29-9295-37ed48e4ded8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754077} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f643cf a685d056-6b50-4e29-9295-37ed48e4ded8 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f643cf a685d056-6b50-4e29-9295-37ed48e4ded8 /tidb/cdc/default/default/changefeed/status/custom-changefeed-name {"checkpoint-ts":457114667952635905,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a685d056-6b50-4e29-9295-37ed48e4ded8 {"id":"a685d056-6b50-4e29-9295-37ed48e4ded8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754077} /tidb/cdc/default/__cdc_meta__/log_coordinator/1f5d95ffd6f643cf a685d056-6b50-4e29-9295-37ed48e4ded8 /tidb/cdc/default/__cdc_meta__/owner/1f5d95ffd6f643cf a685d056-6b50-4e29-9295-37ed48e4ded8 /tidb/cdc/default/default/changefeed/status/custom-changefeed-name {"checkpoint-ts":457114667952635905,"progress":1}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3809.out cli tso query --pd=https://127.0.0.1:2579 cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:07:59 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1045210454.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/sql_mode/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366705458091614 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ec4800011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:6789, start at 2025-04-04 16:08:00.565740841 +0800 CST m=+3.862546650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:00.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:00.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:00.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366705458091614 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ec4800011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:6789, start at 2025-04-04 16:08:00.565740841 +0800 CST m=+3.862546650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:00.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:00.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:00.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366718543011429 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5ec6680003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:6832, start at 2025-04-04 16:08:00.668818618 +0800 CST m=+3.802707296 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:00.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:00.744 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:00.744 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully + set +x + tso='457114669512130562 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114669512130562 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:08:01 CST 2025] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.50795081.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114669551714308 PASS coverage: 1.9% of statements in github.com/pingcap/ticdc/...' + awk -F ' ' '{print $1}' + echo 457114669551714308 PASS coverage: 1.9% of statements in github.com/pingcap/ticdc/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3843.out cli unsafe resolve-lock --region=40 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/batch_add_table/stdout.log no DATA RACE found [Fri Apr 4 16:08:01 CST 2025] <<<<<< run test case batch_add_table success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_G Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Run 6 test table tidb_cdc.syncpoint_v1 exists table tidb_cdc.syncpoint_v1 exists table bdr_mode.finish_mark exists table bdr_mode.finish_mark not exists for 1-th check, retry later table bdr_mode.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/bdr_mode/stdout.log /tmp/tidb_cdc_test/bdr_mode/stdoutdown.log no DATA RACE found [Fri Apr 4 16:07:59 CST 2025] <<<<<< run test case bdr_mode success! >>>>>> PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... [Fri Apr 4 16:08:01 CST 2025] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.51115113.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:02 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/70ea4277-c34a-48db-80dc-5b295f9539a2 {"id":"70ea4277-c34a-48db-80dc-5b295f9539a2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd77a49e5 70ea4277-c34a-48db-80dc-5b295f9539a2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd77a49e5 70ea4277-c34a-48db-80dc-5b295f9539a2' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/70ea4277-c34a-48db-80dc-5b295f9539a2 {"id":"70ea4277-c34a-48db-80dc-5b295f9539a2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd77a49e5 70ea4277-c34a-48db-80dc-5b295f9539a2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd77a49e5 70ea4277-c34a-48db-80dc-5b295f9539a2' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/70ea4277-c34a-48db-80dc-5b295f9539a2 {"id":"70ea4277-c34a-48db-80dc-5b295f9539a2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd77a49e5 70ea4277-c34a-48db-80dc-5b295f9539a2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd77a49e5 70ea4277-c34a-48db-80dc-5b295f9539a2' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.5346.out cli changefeed create --start-ts=457114668947472385 --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: 154391836278655167744404895189314000798 Info: {"upstream_id":7489366693286195187,"id":"154391836278655167744404895189314000798","namespace":"default","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-04T16:08:02.4574831+08:00","start_ts":457114668947472385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114668947472385,"checkpoint_ts":457114668947472385,"checkpoint_time":"2025-04-04 16:07:57.711","gid":{"low":15439183627865516774,"high":4404895189314000798}} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3087 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ecef71c-21ca-4c23-b07f-8f7fe1caeaab {"id":"9ecef71c-21ca-4c23-b07f-8f7fe1caeaab","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17100088867030923722,"high":15578063972551856770},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663257672681,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.125103016+08:00","start-ts":457114667019403266,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668264587271} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668644696096,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ecef71c-21ca-4c23-b07f-8f7fe1caeaab {"id":"9ecef71c-21ca-4c23-b07f-8f7fe1caeaab","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17100088867030923722,"high":15578063972551856770},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663257672681,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.125103016+08:00","start-ts":457114667019403266,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668264587271} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668644696096,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ecef71c-21ca-4c23-b07f-8f7fe1caeaab {"id":"9ecef71c-21ca-4c23-b07f-8f7fe1caeaab","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17100088867030923722,"high":15578063972551856770},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663257672681,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.125103016+08:00","start-ts":457114667019403266,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668264587271} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668644696096,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:08:02 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1027210274.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:02 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8be68447-b82d-461a-beaf-d3a9237af384 {"id":"8be68447-b82d-461a-beaf-d3a9237af384","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8be68447-b82d-461a-beaf-d3a9237af384 {"id":"8be68447-b82d-461a-beaf-d3a9237af384","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8be68447-b82d-461a-beaf-d3a9237af384 {"id":"8be68447-b82d-461a-beaf-d3a9237af384","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114669083000838 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.10130.out cli changefeed create --start-ts=457114669083000838 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489366685609419036,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:03.001296706+08:00","start_ts":457114669083000838,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114669083000838,"checkpoint_ts":457114669083000838,"checkpoint_time":"2025-04-04 16:07:58.228","gid":{"low":13568092162231893258,"high":5558089906852600732}} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19e736ec-beac-400f-a676-3315f181934b {"id":"19e736ec-beac-400f-a676-3315f181934b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754080} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12919736653804788008,"high":4951989450191133113},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663460751938,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.542651828+08:00","start-ts":457114667116658689,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668374949894} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668702892043,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19e736ec-beac-400f-a676-3315f181934b {"id":"19e736ec-beac-400f-a676-3315f181934b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754080} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12919736653804788008,"high":4951989450191133113},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663460751938,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.542651828+08:00","start-ts":457114667116658689,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668374949894} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668702892043,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19e736ec-beac-400f-a676-3315f181934b {"id":"19e736ec-beac-400f-a676-3315f181934b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754080} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12919736653804788008,"high":4951989450191133113},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663460751938,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.542651828+08:00","start-ts":457114667116658689,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668374949894} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668702892043,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:08:02 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1052710529.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3866.out cli unsafe resolve-lock --region=40 --ts=457114669551714308 coverage: 2.7% of statements in github.com/pingcap/ticdc/... PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.cli.8043.out cli tso query --pd=http://127.0.0.1:2379 + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:04 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89a4fe58-5bd0-4203-aa76-935a4bc2e115 {"id":"89a4fe58-5bd0-4203-aa76-935a4bc2e115","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754081} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd78355d5 89a4fe58-5bd0-4203-aa76-935a4bc2e115 /tidb/cdc/default/__cdc_meta__/owner/223195ffd78355d5 89a4fe58-5bd0-4203-aa76-935a4bc2e115' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89a4fe58-5bd0-4203-aa76-935a4bc2e115 {"id":"89a4fe58-5bd0-4203-aa76-935a4bc2e115","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754081} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd78355d5 89a4fe58-5bd0-4203-aa76-935a4bc2e115 /tidb/cdc/default/__cdc_meta__/owner/223195ffd78355d5 89a4fe58-5bd0-4203-aa76-935a4bc2e115' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89a4fe58-5bd0-4203-aa76-935a4bc2e115 {"id":"89a4fe58-5bd0-4203-aa76-935a4bc2e115","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754081} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd78355d5 89a4fe58-5bd0-4203-aa76-935a4bc2e115 /tidb/cdc/default/__cdc_meta__/owner/223195ffd78355d5 89a4fe58-5bd0-4203-aa76-935a4bc2e115' + grep -q 'etcd info' + break + set +x + set +x Command succeeded on attempt 1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:04 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc7827da-3e75-4454-bbbb-e66112e649ff {"id":"fc7827da-3e75-4454-bbbb-e66112e649ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754081} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd77d3ce6 fc7827da-3e75-4454-bbbb-e66112e649ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd77d3ce6 fc7827da-3e75-4454-bbbb-e66112e649ff' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc7827da-3e75-4454-bbbb-e66112e649ff {"id":"fc7827da-3e75-4454-bbbb-e66112e649ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754081} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd77d3ce6 fc7827da-3e75-4454-bbbb-e66112e649ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd77d3ce6 fc7827da-3e75-4454-bbbb-e66112e649ff' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc7827da-3e75-4454-bbbb-e66112e649ff {"id":"fc7827da-3e75-4454-bbbb-e66112e649ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754081} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd77d3ce6 fc7827da-3e75-4454-bbbb-e66112e649ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd77d3ce6 fc7827da-3e75-4454-bbbb-e66112e649ff' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: cdc cli changefeed create --start-ts=457114669512130562 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 1104721870559021912117920069474162227647 Info: {"upstream_id":7489366700864709933,"id":"1104721870559021912117920069474162227647","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:04.373222605+08:00","start_ts":457114669512130562,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114669512130562,"checkpoint_ts":457114669512130562,"checkpoint_time":"2025-04-04 16:07:59.865","gid":{"low":11047218705590219121,"high":17920069474162227647}} Command succeeded on attempt 1 table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.cli.5201.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366734471090795 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5efd440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:6215, start at 2025-04-04 16:08:04.221409772 +0800 CST m=+3.756987305 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:04.230 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:04.228 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:04.228 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366734471090795 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5efd440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:6215, start at 2025-04-04 16:08:04.221409772 +0800 CST m=+3.756987305 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:04.230 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:04.228 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:04.228 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366734078855025 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5efc90000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:6277, start at 2025-04-04 16:08:04.153217304 +0800 CST m=+3.532615029 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:04.162 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:04.216 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:04.216 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + set +x + tso='457114670501199881 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114670501199881 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:08:05 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.80678069.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3668 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 {"id":"6fcd4a89-a0ea-4b8d-b43e-b75bc24de890","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754082} /tidb/cdc/default/__cdc_meta__/capture/9ecef71c-21ca-4c23-b07f-8f7fe1caeaab {"id":"9ecef71c-21ca-4c23-b07f-8f7fe1caeaab","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0c28 6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0c28 6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17100088867030923722,"high":15578063972551856770},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663257672681,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.125103016+08:00","start-ts":457114667019403266,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668264587271} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668644696096,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 {"id":"6fcd4a89-a0ea-4b8d-b43e-b75bc24de890","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754082} /tidb/cdc/default/__cdc_meta__/capture/9ecef71c-21ca-4c23-b07f-8f7fe1caeaab {"id":"9ecef71c-21ca-4c23-b07f-8f7fe1caeaab","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0c28 6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0c28 6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17100088867030923722,"high":15578063972551856770},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663257672681,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.125103016+08:00","start-ts":457114667019403266,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668264587271} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668644696096,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 {"id":"6fcd4a89-a0ea-4b8d-b43e-b75bc24de890","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754082} /tidb/cdc/default/__cdc_meta__/capture/9ecef71c-21ca-4c23-b07f-8f7fe1caeaab {"id":"9ecef71c-21ca-4c23-b07f-8f7fe1caeaab","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0ae5 e69263c1-210d-4131-901b-5355d564527f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0c28 6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0c28 6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17100088867030923722,"high":15578063972551856770},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663257672681,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.125103016+08:00","start-ts":457114667019403266,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668264587271} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668644696096,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.10346.out cli capture list table fail_over_ddl_test.test2 not exists for 1-th check, retry later Starting Upstream TiDB... Verifying downstream PD is started... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19e736ec-beac-400f-a676-3315f181934b {"id":"19e736ec-beac-400f-a676-3315f181934b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754080} /tidb/cdc/default/__cdc_meta__/capture/cc0e4a6c-c0f8-4972-af62-12c677ee15b2 {"id":"cc0e4a6c-c0f8-4972-af62-12c677ee15b2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754083} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12919736653804788008,"high":4951989450191133113},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663460751938,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.542651828+08:00","start-ts":457114667116658689,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668374949894} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668768165911,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19e736ec-beac-400f-a676-3315f181934b {"id":"19e736ec-beac-400f-a676-3315f181934b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754080} /tidb/cdc/default/__cdc_meta__/capture/cc0e4a6c-c0f8-4972-af62-12c677ee15b2 {"id":"cc0e4a6c-c0f8-4972-af62-12c677ee15b2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754083} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12919736653804788008,"high":4951989450191133113},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663460751938,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.542651828+08:00","start-ts":457114667116658689,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668374949894} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668768165911,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19e736ec-beac-400f-a676-3315f181934b {"id":"19e736ec-beac-400f-a676-3315f181934b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754080} /tidb/cdc/default/__cdc_meta__/capture/cc0e4a6c-c0f8-4972-af62-12c677ee15b2 {"id":"cc0e4a6c-c0f8-4972-af62-12c677ee15b2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754083} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f5e2 c9d7f8b9-d783-41b8-80ac-cfecf6b026b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f6c6 19e736ec-beac-400f-a676-3315f181934b /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12919736653804788008,"high":4951989450191133113},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663460751938,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.542651828+08:00","start-ts":457114667116658689,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668374949894} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114668768165911,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.10599.out cli capture list table cdc_tiflash_test.multi_data_type exists + set +x + tso='457114670868201479 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114670868201479 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x table test.table1 not exists for 1-th check, retry later check diff failed 1-th time, retry later check diff failed 3-th time, retry later + set +x + ans='[ { "id": "9ecef71c-21ca-4c23-b07f-8f7fe1caeaab", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "6fcd4a89-a0ea-4b8d-b43e-b75bc24de890", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"9ecef71c-21ca-4c23-b07f-8f7fe1caeaab",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"6fcd4a89-a0ea-4b8d-b43e-b75bc24de890",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 + '[' -z 6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.10373.out cli changefeed move-table -c test -t 114 -d 6fcd4a89-a0ea-4b8d-b43e-b75bc24de890 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.7480.out cli tso query --pd=http://127.0.0.1:2379 + set +x + ans='[ { "id": "19e736ec-beac-400f-a676-3315f181934b", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "cc0e4a6c-c0f8-4972-af62-12c677ee15b2", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"19e736ec-beac-400f-a676-3315f181934b",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"cc0e4a6c-c0f8-4972-af62-12c677ee15b2",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=cc0e4a6c-c0f8-4972-af62-12c677ee15b2 + '[' -z cc0e4a6c-c0f8-4972-af62-12c677ee15b2 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d cc0e4a6c-c0f8-4972-af62-12c677ee15b2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.10630.out cli changefeed move-table -c test -t 114 -d cc0e4a6c-c0f8-4972-af62-12c677ee15b2 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_attributes.attributes_t1_new not exists for 1-th check, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 16266 0 --:--:-- --:--:-- --:--:-- 16266 { "version": "v9.0.0-beta.1-55-g52a1009d", "git_hash": "52a1009d7bcd84b283c5443601c9b3160b35bb7c", "id": "a685d056-6b50-4e29-9295-37ed48e4ded8", "cluster_id": "default", "pid": 3710, "is_owner": true, "liveness": 0 }cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/cli_tls_with_auth/stdout_cli_tls_with_auth_tls1.log /tmp/tidb_cdc_test/cli_tls_with_auth/stdout_cli_tls_with_auth_tls1_restart.log no DATA RACE found [Fri Apr 4 16:08:08 CST 2025] <<<<<< run test case cli_tls_with_auth success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:08 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/263eb2d5-d16c-4bf7-94d5-1fec4536102a {"id":"263eb2d5-d16c-4bf7-94d5-1fec4536102a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754085} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/263eb2d5-d16c-4bf7-94d5-1fec4536102a {"id":"263eb2d5-d16c-4bf7-94d5-1fec4536102a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754085} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/263eb2d5-d16c-4bf7-94d5-1fec4536102a {"id":"263eb2d5-d16c-4bf7-94d5-1fec4536102a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754085} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114670501199881 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.cli.8131.out cli changefeed create --start-ts=457114670501199881 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489366705458091614,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:08.436061797+08:00","start_ts":457114670501199881,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114670501199881,"checkpoint_ts":457114670501199881,"checkpoint_time":"2025-04-04 16:08:03.638","gid":{"low":4561607153064955909,"high":2610855494131874458}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10068) - No such process wait process 10068 exit for 1-th time... table test.table1 exists table test.table2 exists check diff successfully process 10068 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:08:08 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1021110213.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later check diff successfully + set +x + tso='457114671494725633 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114671494725633 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/tiflash/stdout.log no DATA RACE found [Fri Apr 4 16:08:09 CST 2025] <<<<<< run test case tiflash success! >>>>>> table ddl_attributes.attributes_t1_new not exists for 2-th check, retry later + set +x Command succeeded on attempt 1 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_K/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Fri Apr 4 16:08:09 CST 2025] <<<<<< START cdc server in server_config_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.75337535.out server --log-file /tmp/tidb_cdc_test/server_config_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/server_config_compatibility/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/server_config_compatibility/conf/server.toml + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366762938365563 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5f4f440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:6875, start at 2025-04-04 16:08:09.430401568 +0800 CST m=+3.252190983 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:09.438 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:09.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:09.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366762938365563 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5f4f440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:6875, start at 2025-04-04 16:08:09.430401568 +0800 CST m=+3.252190983 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:09.438 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:09.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:09.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366762330757888 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5f52d80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:6955, start at 2025-04-04 16:08:09.660540592 +0800 CST m=+3.328825000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:09.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:09.654 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:09.654 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/error.log check diff successfully arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.cli.5371.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/batch_update_to_no_batch/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":1}' + echo '{"count":1}' {"count":1} ++ echo '{"count":1}' ++ jq -r .count + value=1 + '[' -1 == -1 ']' + '[' 1 '!=' null ']' + exit 0 [Fri Apr 4 16:08:09 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.60456047.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3187684-d30a-4462-a825-f5bc626daa9d {"id":"f3187684-d30a-4462-a825-f5bc626daa9d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754088} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13568092162231893258,"high":5558089906852600732},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366685609419036,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:03.001296706+08:00","start-ts":457114669083000838,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114670328184839} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114670839365656,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3187684-d30a-4462-a825-f5bc626daa9d {"id":"f3187684-d30a-4462-a825-f5bc626daa9d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754088} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13568092162231893258,"high":5558089906852600732},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366685609419036,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:03.001296706+08:00","start-ts":457114669083000838,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114670328184839} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114670839365656,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3187684-d30a-4462-a825-f5bc626daa9d {"id":"f3187684-d30a-4462-a825-f5bc626daa9d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754088} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13568092162231893258,"high":5558089906852600732},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366685609419036,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:03.001296706+08:00","start-ts":457114669083000838,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114670328184839} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114670839365656,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:08:11 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1028710289.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_attributes.attributes_t1_new not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a80e8294-6489-4260-a161-9d2535022476 {"id":"a80e8294-6489-4260-a161-9d2535022476","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754089} /tidb/cdc/default/__cdc_meta__/capture/a9727a03-ef64-41f9-84e2-c17dadf4d077 {"id":"a9727a03-ef64-41f9-84e2-c17dadf4d077","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754049} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc1d58 a80e8294-6489-4260-a161-9d2535022476 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc1d58 a80e8294-6489-4260-a161-9d2535022476 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5350837396807433449,"high":14629577223381727151},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366517910410119,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:22.351134378+08:00","start-ts":457114658422652929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114659667836938} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114671870640131,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a80e8294-6489-4260-a161-9d2535022476 {"id":"a80e8294-6489-4260-a161-9d2535022476","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754089} /tidb/cdc/default/__cdc_meta__/capture/a9727a03-ef64-41f9-84e2-c17dadf4d077 {"id":"a9727a03-ef64-41f9-84e2-c17dadf4d077","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754049} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc1d58 a80e8294-6489-4260-a161-9d2535022476 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc1d58 a80e8294-6489-4260-a161-9d2535022476 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5350837396807433449,"high":14629577223381727151},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366517910410119,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:22.351134378+08:00","start-ts":457114658422652929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114659667836938} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114671870640131,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a80e8294-6489-4260-a161-9d2535022476 {"id":"a80e8294-6489-4260-a161-9d2535022476","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754089} /tidb/cdc/default/__cdc_meta__/capture/a9727a03-ef64-41f9-84e2-c17dadf4d077 {"id":"a9727a03-ef64-41f9-84e2-c17dadf4d077","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754049} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd6dc1d58 a80e8294-6489-4260-a161-9d2535022476 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc182a a9727a03-ef64-41f9-84e2-c17dadf4d077 /tidb/cdc/default/__cdc_meta__/owner/223195ffd6dc1d58 a80e8294-6489-4260-a161-9d2535022476 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5350837396807433449,"high":14629577223381727151},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366517910410119,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:22.351134378+08:00","start-ts":457114658422652929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114659667836938} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114671870640131,"progress":1}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.cli.8155.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later + set +x + tso='457114672506601476 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114672506601476 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.cli.5415.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_K Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:12 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/990075d0-80fe-49eb-950b-70b916d2398c {"id":"990075d0-80fe-49eb-950b-70b916d2398c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754090} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd79eabfc 990075d0-80fe-49eb-950b-70b916d2398c /tidb/cdc/default/__cdc_meta__/owner/223195ffd79eabfc 990075d0-80fe-49eb-950b-70b916d2398c' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/990075d0-80fe-49eb-950b-70b916d2398c {"id":"990075d0-80fe-49eb-950b-70b916d2398c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754090} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd79eabfc 990075d0-80fe-49eb-950b-70b916d2398c /tidb/cdc/default/__cdc_meta__/owner/223195ffd79eabfc 990075d0-80fe-49eb-950b-70b916d2398c' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/990075d0-80fe-49eb-950b-70b916d2398c {"id":"990075d0-80fe-49eb-950b-70b916d2398c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754090} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd79eabfc 990075d0-80fe-49eb-950b-70b916d2398c /tidb/cdc/default/__cdc_meta__/owner/223195ffd79eabfc 990075d0-80fe-49eb-950b-70b916d2398c' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.7595.out cli changefeed create --start-ts=457114671494725633 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 1378104741996891609017702652136804827301 Info: {"upstream_id":7489366734471090795,"id":"1378104741996891609017702652136804827301","namespace":"default","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:13.259035517+08:00","start_ts":457114671494725633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114671494725633,"checkpoint_ts":457114671494725633,"checkpoint_time":"2025-04-04 16:08:07.428","gid":{"low":13781047419968916090,"high":17702652136804827301}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... table fail_over_ddl_test.test2 exists table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark exists command: run_sql 'show create table ddl_attributes.placement_t1;' 127.0.0.1 4000 run task successfully command: run_sql 'show create table ddl_attributes.placement_t2;' 127.0.0.1 4000 run task successfully command: run_sql 'show create table ddl_attributes.placement_t1;' 127.0.0.1 3306 && check_contains 'CREATE TABLE `placement_t1`' run task successfully command: run_sql 'show create table ddl_attributes.placement_t2;' 127.0.0.1 3306 && check_contains 'CREATE TABLE `placement_t2`' run task successfully command: run_sql 'show create table ddl_attributes.ttl_t1;' 127.0.0.1 3306 && check_contains /*T![ttl] TTL=\`t\` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE=OFF */ /*T![ttl] TTL_JOB_INTERVAL=1h */ run task successfully command: run_sql 'show create table ddl_attributes.ttl_t2;' 127.0.0.1 3306 && check_contains /*T![ttl] TTL=\`t\` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE=OFF */ /*T![ttl] TTL_JOB_INTERVAL=1h */ TEST FAILED: OUTPUT DOES NOT CONTAIN '/*T![ttl]' ____________________________________ *************************** 1. row *************************** Table: ttl_t2 Create Table: CREATE TABLE `ttl_t2` ( `id` bigint NOT NULL AUTO_INCREMENT, `t` datetime DEFAULT NULL, PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8072) - No such process wait process 8072 exit for 1-th time... process 8072 already exit [Fri Apr 4 16:08:13 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.82138215.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366767214488677 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5f91100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:6440, start at 2025-04-04 16:08:13.670548408 +0800 CST m=+3.538786900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:13.732 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:13.735 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:13.735 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366767214488677 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5f91100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:6440, start at 2025-04-04 16:08:13.670548408 +0800 CST m=+3.538786900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:13.732 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:13.735 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:13.735 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366772421872367 TiDB Cluster ID. tikv_gc_leader_uuid 657ff5f95bc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:6514, start at 2025-04-04 16:08:13.956648085 +0800 CST m=+3.696722690 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:13.965 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:13.935 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:13.935 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... + set +x + tso='457114672981082113 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114672981082113 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:08:14 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.81848186.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + set +x + tso='457114672991567875 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114672991567875 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x Resume changefeed with checkpointTs3 457114672991567875 table test.table5 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a0edd26-02ee-44c4-b17c-283d9a30b064 {"id":"8a0edd26-02ee-44c4-b17c-283d9a30b064","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754091} /tidb/cdc/default/__cdc_meta__/capture/f3187684-d30a-4462-a825-f5bc626daa9d {"id":"f3187684-d30a-4462-a825-f5bc626daa9d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754088} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741c56 8a0edd26-02ee-44c4-b17c-283d9a30b064 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741c56 8a0edd26-02ee-44c4-b17c-283d9a30b064 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13568092162231893258,"high":5558089906852600732},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366685609419036,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:03.001296706+08:00","start-ts":457114669083000838,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114670328184839} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114670839365656,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a0edd26-02ee-44c4-b17c-283d9a30b064 {"id":"8a0edd26-02ee-44c4-b17c-283d9a30b064","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754091} /tidb/cdc/default/__cdc_meta__/capture/f3187684-d30a-4462-a825-f5bc626daa9d {"id":"f3187684-d30a-4462-a825-f5bc626daa9d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754088} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741c56 8a0edd26-02ee-44c4-b17c-283d9a30b064 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741c56 8a0edd26-02ee-44c4-b17c-283d9a30b064 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13568092162231893258,"high":5558089906852600732},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366685609419036,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:03.001296706+08:00","start-ts":457114669083000838,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114670328184839} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114670839365656,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a0edd26-02ee-44c4-b17c-283d9a30b064 {"id":"8a0edd26-02ee-44c4-b17c-283d9a30b064","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754091} /tidb/cdc/default/__cdc_meta__/capture/f3187684-d30a-4462-a825-f5bc626daa9d {"id":"f3187684-d30a-4462-a825-f5bc626daa9d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754088} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741aec 8be68447-b82d-461a-beaf-d3a9237af384 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741c56 8a0edd26-02ee-44c4-b17c-283d9a30b064 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741c56 8a0edd26-02ee-44c4-b17c-283d9a30b064 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13568092162231893258,"high":5558089906852600732},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366685609419036,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:03.001296706+08:00","start-ts":457114669083000838,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114670328184839} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114670839365656,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.10362.out cli capture list check diff failed 2-th time, retry later command: run_sql 'show create table ddl_attributes.ttl_t2;' 127.0.0.1 3306 && check_contains /*T![ttl] TTL=\`t\` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE=OFF */ /*T![ttl] TTL_JOB_INTERVAL=1h */ run task successfully command: run_sql 'show create table ddl_attributes.ttl_t3;' 127.0.0.1 3306 && check_contains /*T![ttl] TTL=\`t\` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE=OFF */ /*T![ttl] TTL_JOB_INTERVAL=1h */ run task successfully command: run_sql 'show create table ddl_attributes.ttl_t4;' 127.0.0.1 3306 && check_not_contains 'TTL_ENABLE' && check_not_contains 'TTL=' run task successfully command: run_sql 'show create table ddl_attributes.ttl_t5;' 127.0.0.1 3306 && check_contains /*T![ttl] TTL=\`t\` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE=OFF */ /*T![ttl] TTL_JOB_INTERVAL=7h */ run task successfully check diff successfully + set +x + ans='[ { "id": "f3187684-d30a-4462-a825-f5bc626daa9d", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "8a0edd26-02ee-44c4-b17c-283d9a30b064", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"f3187684-d30a-4462-a825-f5bc626daa9d",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"8a0edd26-02ee-44c4-b17c-283d9a30b064",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=8a0edd26-02ee-44c4-b17c-283d9a30b064 + '[' -z 8a0edd26-02ee-44c4-b17c-283d9a30b064 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 8a0edd26-02ee-44c4-b17c-283d9a30b064 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.10394.out cli changefeed move-table -c test -t 114 -d 8a0edd26-02ee-44c4-b17c-283d9a30b064 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/ddl_attributes/stdout.log no DATA RACE found [Fri Apr 4 16:08:16 CST 2025] <<<<<< run test case ddl_attributes success! >>>>>> table test.table5 exists table test.table6 exists + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 check diff failed 1-th time, retry later + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.7800.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "readallfields"="true" "readproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "dotransactions"="false" "insertproportion"="0" "updateproportion"="0" "mysql.port"="4000" "threadcount"="2" "recordcount"="10" "operationcount"="0" "requestdistribution"="uniform" "mysql.user"="root" "mysql.db"="sink_retry" ********************************************** Run finished, takes 4.750104ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3570.1, Avg(us): 881, Min(us): 432, Max(us): 2193, 95th(us): 3000, 99th(us): 3000 check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:16 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/458433fc-0359-4bd3-b843-6a8d50123b3c {"id":"458433fc-0359-4bd3-b843-6a8d50123b3c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754094} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114672270671877,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/458433fc-0359-4bd3-b843-6a8d50123b3c {"id":"458433fc-0359-4bd3-b843-6a8d50123b3c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754094} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114672270671877,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/458433fc-0359-4bd3-b843-6a8d50123b3c {"id":"458433fc-0359-4bd3-b843-6a8d50123b3c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754094} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114672270671877,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:08:16 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.82918293.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check data successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/server_config_compatibility/stdout.log no DATA RACE found [Fri Apr 4 16:08:17 CST 2025] <<<<<< run test case server_config_compatibility success! >>>>>> cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseH-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:17 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c65255a7-4e05-470b-8f82-9e32b067a6b6 {"id":"c65255a7-4e05-470b-8f82-9e32b067a6b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754094} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c65255a7-4e05-470b-8f82-9e32b067a6b6 {"id":"c65255a7-4e05-470b-8f82-9e32b067a6b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754094} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c65255a7-4e05-470b-8f82-9e32b067a6b6 {"id":"c65255a7-4e05-470b-8f82-9e32b067a6b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754094} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114672981082113 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.cli.8245.out cli changefeed create --start-ts=457114672981082113 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489366762938365563,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:17.872395277+08:00","start_ts":457114672981082113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114672981082113,"checkpoint_ts":457114672981082113,"checkpoint_time":"2025-04-04 16:08:13.098","gid":{"low":1676362822815118304,"high":13189067738902776737}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x + tso='457114674065571842 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114674065571842 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114674065571842 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Fri Apr 4 16:08:18 CST 2025] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.78407842.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/multi_rocks/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x Command succeeded on attempt 1 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully Resume changefeed with checkpointTs1 457114670868201479 table test.table3 not exists for 1-th check, retry later table fail_over_ddl_test.test2 not exists for 1-th check, retry later command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05b6001f-5912-4687-a3ca-dc74c7580a32 {"id":"05b6001f-5912-4687-a3ca-dc74c7580a32","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754097} /tidb/cdc/default/__cdc_meta__/capture/458433fc-0359-4bd3-b843-6a8d50123b3c {"id":"458433fc-0359-4bd3-b843-6a8d50123b3c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754094} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899e56 05b6001f-5912-4687-a3ca-dc74c7580a32 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899e56 05b6001f-5912-4687-a3ca-dc74c7580a32 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114672270671877,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05b6001f-5912-4687-a3ca-dc74c7580a32 {"id":"05b6001f-5912-4687-a3ca-dc74c7580a32","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754097} /tidb/cdc/default/__cdc_meta__/capture/458433fc-0359-4bd3-b843-6a8d50123b3c {"id":"458433fc-0359-4bd3-b843-6a8d50123b3c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754094} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899e56 05b6001f-5912-4687-a3ca-dc74c7580a32 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899e56 05b6001f-5912-4687-a3ca-dc74c7580a32 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114672270671877,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05b6001f-5912-4687-a3ca-dc74c7580a32 {"id":"05b6001f-5912-4687-a3ca-dc74c7580a32","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754097} /tidb/cdc/default/__cdc_meta__/capture/458433fc-0359-4bd3-b843-6a8d50123b3c {"id":"458433fc-0359-4bd3-b843-6a8d50123b3c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754094} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899cec 263eb2d5-d16c-4bf7-94d5-1fec4536102a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899e56 05b6001f-5912-4687-a3ca-dc74c7580a32 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/owner/223195ffd7899e56 05b6001f-5912-4687-a3ca-dc74c7580a32 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114672270671877,"progress":1}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table3 exists table test.table4 exists table fail_over_ddl_test.test2 exists check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8189) - No such process wait process 8189 exit for 1-th time... process 8189 already exit [Fri Apr 4 16:08:21 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.83168318.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:21 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/996bad3d-4c08-4804-8656-b48c6e221551 {"id":"996bad3d-4c08-4804-8656-b48c6e221551","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754099} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7c04cee 996bad3d-4c08-4804-8656-b48c6e221551 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7c04cee 996bad3d-4c08-4804-8656-b48c6e221551' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/996bad3d-4c08-4804-8656-b48c6e221551 {"id":"996bad3d-4c08-4804-8656-b48c6e221551","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754099} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7c04cee 996bad3d-4c08-4804-8656-b48c6e221551 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7c04cee 996bad3d-4c08-4804-8656-b48c6e221551' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/996bad3d-4c08-4804-8656-b48c6e221551 {"id":"996bad3d-4c08-4804-8656-b48c6e221551","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754099} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7c04cee 996bad3d-4c08-4804-8656-b48c6e221551 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7c04cee 996bad3d-4c08-4804-8656-b48c6e221551' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114674065571842 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.7915.out cli changefeed create --start-ts=457114674065571842 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489366767214488677,"id":"test-1","namespace":"default","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T16:08:22.041131925+08:00","start_ts":457114674065571842,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114674065571842,"checkpoint_ts":457114674065571842,"checkpoint_time":"2025-04-04 16:08:17.235","gid":{"low":12558014150228006066,"high":13433598636595863470}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10292) - No such process wait process 10292 exit for 1-th time... process 10292 already exit [Fri Apr 4 16:08:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1045110453.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 + '[' mysql == mysql ']' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later Run 7 test VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366817178628491 TiDB Cluster ID. tikv_gc_leader_uuid 657ff601bf40011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:6863, start at 2025-04-04 16:08:22.546304026 +0800 CST m=+3.396320887 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:22.553 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:22.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:22.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366817178628491 TiDB Cluster ID. tikv_gc_leader_uuid 657ff601bf40011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:6863, start at 2025-04-04 16:08:22.546304026 +0800 CST m=+3.396320887 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:22.553 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:22.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:22.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366817362494389 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6021300003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:6929, start at 2025-04-04 16:08:22.86369208 +0800 CST m=+3.602352867 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:22.872 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:22.860 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:22.860 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 4 16:08:24 CST 2025] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/logservice/schemastore/getAllPhysicalTablesGCFastFail=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.57265728.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366821928375898 TiDB Cluster ID. tikv_gc_leader_uuid 657ff602f000002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:6414, start at 2025-04-04 16:08:23.746415882 +0800 CST m=+3.401682377 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:23.753 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:23.744 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:23.744 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366821928375898 TiDB Cluster ID. tikv_gc_leader_uuid 657ff602f000002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:6414, start at 2025-04-04 16:08:23.746415882 +0800 CST m=+3.401682377 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:23.753 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:23.744 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:23.744 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366820991005152 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6031d4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:6472, start at 2025-04-04 16:08:23.960715091 +0800 CST m=+3.507147691 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:23.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:23.975 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:23.975 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad029377-c647-4ed5-9b3a-5a8749f93c6a {"id":"ad029377-c647-4ed5-9b3a-5a8749f93c6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754102} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114674606637084,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad029377-c647-4ed5-9b3a-5a8749f93c6a {"id":"ad029377-c647-4ed5-9b3a-5a8749f93c6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754102} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114674606637084,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad029377-c647-4ed5-9b3a-5a8749f93c6a {"id":"ad029377-c647-4ed5-9b3a-5a8749f93c6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754102} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114674606637084,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:08:24 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.83948396.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_H Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3376e15d-c023-42db-afe0-26c3f8b889f6 {"id":"3376e15d-c023-42db-afe0-26c3f8b889f6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754102} /tidb/cdc/default/__cdc_meta__/capture/f3187684-d30a-4462-a825-f5bc626daa9d {"id":"f3187684-d30a-4462-a825-f5bc626daa9d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754088} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741e58 3376e15d-c023-42db-afe0-26c3f8b889f6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741e58 3376e15d-c023-42db-afe0-26c3f8b889f6 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13568092162231893258,"high":5558089906852600732},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366685609419036,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:03.001296706+08:00","start-ts":457114669083000838,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114670328184839} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114675112312848,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3376e15d-c023-42db-afe0-26c3f8b889f6 {"id":"3376e15d-c023-42db-afe0-26c3f8b889f6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754102} /tidb/cdc/default/__cdc_meta__/capture/f3187684-d30a-4462-a825-f5bc626daa9d {"id":"f3187684-d30a-4462-a825-f5bc626daa9d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754088} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741e58 3376e15d-c023-42db-afe0-26c3f8b889f6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741e58 3376e15d-c023-42db-afe0-26c3f8b889f6 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13568092162231893258,"high":5558089906852600732},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366685609419036,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:03.001296706+08:00","start-ts":457114669083000838,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114670328184839} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114675112312848,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3376e15d-c023-42db-afe0-26c3f8b889f6 {"id":"3376e15d-c023-42db-afe0-26c3f8b889f6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754102} /tidb/cdc/default/__cdc_meta__/capture/f3187684-d30a-4462-a825-f5bc626daa9d {"id":"f3187684-d30a-4462-a825-f5bc626daa9d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754088} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7741e58 3376e15d-c023-42db-afe0-26c3f8b889f6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741bf0 f3187684-d30a-4462-a825-f5bc626daa9d /tidb/cdc/default/__cdc_meta__/owner/223195ffd7741e58 3376e15d-c023-42db-afe0-26c3f8b889f6 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13568092162231893258,"high":5558089906852600732},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366685609419036,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:03.001296706+08:00","start-ts":457114669083000838,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114670328184839} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114675112312848,"progress":1}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... table test.t1 exists + sleep 10 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.cli.8189.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/multi_cdc_cluster/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_J/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='457114676377419777 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114676377419777 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:08:27 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.82248226.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:27 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3559 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6daf9681-b9af-4c7c-b106-cab3be260112 {"id":"6daf9681-b9af-4c7c-b106-cab3be260112","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754104} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd78355d5 89a4fe58-5bd0-4203-aa76-935a4bc2e115 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd78358c2 6daf9681-b9af-4c7c-b106-cab3be260112 /tidb/cdc/default/__cdc_meta__/owner/223195ffd78358c2 6daf9681-b9af-4c7c-b106-cab3be260112 /tidb/cdc/default/default/changefeed/info/1364080071926037623110010353592831602866 {"id":{"id":{"low":13640800719260376231,"high":10010353592831602866},"display":{"name":"1364080071926037623110010353592831602866","namespace":"default"}},"upstream-id":7489366707262181732,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:08:04.693095431+08:00","start-ts":457114670763343878,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2025-04-04T16:08:26.264949061+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrSnapshotLostByGC","message":"[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)"},"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674603753477} /tidb/cdc/default/default/changefeed/status/1364080071926037623110010353592831602866 {"checkpoint-ts":457114675337756677,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6daf9681-b9af-4c7c-b106-cab3be260112 {"id":"6daf9681-b9af-4c7c-b106-cab3be260112","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754104} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd78355d5 89a4fe58-5bd0-4203-aa76-935a4bc2e115 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd78358c2 6daf9681-b9af-4c7c-b106-cab3be260112 /tidb/cdc/default/__cdc_meta__/owner/223195ffd78358c2 6daf9681-b9af-4c7c-b106-cab3be260112 /tidb/cdc/default/default/changefeed/info/1364080071926037623110010353592831602866 {"id":{"id":{"low":13640800719260376231,"high":10010353592831602866},"display":{"name":"1364080071926037623110010353592831602866","namespace":"default"}},"upstream-id":7489366707262181732,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:08:04.693095431+08:00","start-ts":457114670763343878,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2025-04-04T16:08:26.264949061+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrSnapshotLostByGC","message":"[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)"},"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674603753477} /tidb/cdc/default/default/changefeed/status/1364080071926037623110010353592831602866 {"checkpoint-ts":457114675337756677,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6daf9681-b9af-4c7c-b106-cab3be260112 {"id":"6daf9681-b9af-4c7c-b106-cab3be260112","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754104} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd78355d5 89a4fe58-5bd0-4203-aa76-935a4bc2e115 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd78358c2 6daf9681-b9af-4c7c-b106-cab3be260112 /tidb/cdc/default/__cdc_meta__/owner/223195ffd78358c2 6daf9681-b9af-4c7c-b106-cab3be260112 /tidb/cdc/default/default/changefeed/info/1364080071926037623110010353592831602866 {"id":{"id":{"low":13640800719260376231,"high":10010353592831602866},"display":{"name":"1364080071926037623110010353592831602866","namespace":"default"}},"upstream-id":7489366707262181732,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:08:04.693095431+08:00","start-ts":457114670763343878,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2025-04-04T16:08:26.264949061+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrSnapshotLostByGC","message":"[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)"},"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674603753477} /tidb/cdc/default/default/changefeed/status/1364080071926037623110010353592831602866 {"checkpoint-ts":457114675337756677,"progress":1}' + grep -q 'etcd info' + break + set +x command: check_changefeed_state http://127.0.0.1:2379 1364080071926037623110010353592831602866 failed ErrSnapshotLostByGC + endpoints=http://127.0.0.1:2379 + changefeed_id=1364080071926037623110010353592831602866 + expected_state=failed + error_msg=ErrSnapshotLostByGC + tls_dir=ErrSnapshotLostByGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c 1364080071926037623110010353592831602866 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366707262181732, "id": "1364080071926037623110010353592831602866", "namespace": "default", "state": "failed", "checkpoint_tso": 457114675337756677, "checkpoint_time": "2025-04-04 16:08:22.088", "error": { "time": "2025-04-04T16:08:26.264949061+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)" } }' + echo '{ "upstream_id": 7489366707262181732, "id": "1364080071926037623110010353592831602866", "namespace": "default", "state": "failed", "checkpoint_tso": 457114675337756677, "checkpoint_time": "2025-04-04 16:08:22.088", "error": { "time": "2025-04-04T16:08:26.264949061+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)" } }' { "upstream_id": 7489366707262181732, "id": "1364080071926037623110010353592831602866", "namespace": "default", "state": "failed", "checkpoint_tso": 457114675337756677, "checkpoint_time": "2025-04-04 16:08:22.088", "error": { "time": "2025-04-04T16:08:26.264949061+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7489366707262181732, '"id":' '"1364080071926037623110010353592831602866",' '"namespace":' '"default",' '"state":' '"failed",' '"checkpoint_tso":' 457114675337756677, '"checkpoint_time":' '"2025-04-04' '16:08:22.088",' '"error":' '{' '"time":' '"2025-04-04T16:08:26.264949061+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts '%!d(string=snapTs' %d is smaller than gcTs '%d)' is earlier than or equal to GC safepoint at '0%!(EXTRA' 'uint64=457114511858728960)"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489366707262181732, '"id":' '"1364080071926037623110010353592831602866",' '"namespace":' '"default",' '"state":' '"failed",' '"checkpoint_tso":' 457114675337756677, '"checkpoint_time":' '"2025-04-04' '16:08:22.088",' '"error":' '{' '"time":' '"2025-04-04T16:08:26.264949061+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts '%!d(string=snapTs' %d is smaller than gcTs '%d)' is earlier than or equal to GC safepoint at '0%!(EXTRA' 'uint64=457114511858728960)"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960) =~ ErrSnapshotLostByGC ]] run task successfully command: check_changefeed_state http://127.0.0.1:2379 1364080071926037623110010353592831602866 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=1364080071926037623110010353592831602866 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c 1364080071926037623110010353592831602866 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366707262181732, "id": "1364080071926037623110010353592831602866", "namespace": "default", "state": "normal", "checkpoint_tso": 457114676740227075, "checkpoint_time": "2025-04-04 16:08:27.438", "error": { "time": "2025-04-04T16:08:26.264949061+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)" } }' + echo '{ "upstream_id": 7489366707262181732, "id": "1364080071926037623110010353592831602866", "namespace": "default", "state": "normal", "checkpoint_tso": 457114676740227075, "checkpoint_time": "2025-04-04 16:08:27.438", "error": { "time": "2025-04-04T16:08:26.264949061+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)" } }' { "upstream_id": 7489366707262181732, "id": "1364080071926037623110010353592831602866", "namespace": "default", "state": "normal", "checkpoint_tso": 457114676740227075, "checkpoint_time": "2025-04-04 16:08:27.438", "error": { "time": "2025-04-04T16:08:26.264949061+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7489366707262181732, '"id":' '"1364080071926037623110010353592831602866",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114676740227075, '"checkpoint_time":' '"2025-04-04' '16:08:27.438",' '"error":' '{' '"time":' '"2025-04-04T16:08:26.264949061+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts '%!d(string=snapTs' %d is smaller than gcTs '%d)' is earlier than or equal to GC safepoint at '0%!(EXTRA' 'uint64=457114511858728960)"' '}' '}' + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489366707262181732, '"id":' '"1364080071926037623110010353592831602866",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114676740227075, '"checkpoint_time":' '"2025-04-04' '16:08:27.438",' '"error":' '{' '"time":' '"2025-04-04T16:08:26.264949061+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts '%!d(string=snapTs' %d is smaller than gcTs '%d)' is earlier than or equal to GC safepoint at '0%!(EXTRA' 'uint64=457114511858728960)"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960) =~ null ]] + echo 'error message '\''[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)'\'' is not as expected '\''null'\''' error message '[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts %!d(string=snapTs %d is smaller than gcTs %d) is earlier than or equal to GC safepoint at 0%!(EXTRA uint64=457114511858728960)' is not as expected 'null' + exit 1 run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10277) - No such process wait process 10277 exit for 1-th time... process 10277 already exit [Fri Apr 4 16:08:25 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1044510447.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3565 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/491edc74-0e51-4ac6-9569-c8c1fbc04b15 {"id":"491edc74-0e51-4ac6-9569-c8c1fbc04b15","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754105} /tidb/cdc/default/__cdc_meta__/capture/9ecef71c-21ca-4c23-b07f-8f7fe1caeaab {"id":"9ecef71c-21ca-4c23-b07f-8f7fe1caeaab","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a1086 491edc74-0e51-4ac6-9569-c8c1fbc04b15 /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a1086 491edc74-0e51-4ac6-9569-c8c1fbc04b15 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17100088867030923722,"high":15578063972551856770},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663257672681,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.125103016+08:00","start-ts":457114667019403266,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668264587271} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114672026353675,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/491edc74-0e51-4ac6-9569-c8c1fbc04b15 {"id":"491edc74-0e51-4ac6-9569-c8c1fbc04b15","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754105} /tidb/cdc/default/__cdc_meta__/capture/9ecef71c-21ca-4c23-b07f-8f7fe1caeaab {"id":"9ecef71c-21ca-4c23-b07f-8f7fe1caeaab","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a1086 491edc74-0e51-4ac6-9569-c8c1fbc04b15 /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a1086 491edc74-0e51-4ac6-9569-c8c1fbc04b15 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17100088867030923722,"high":15578063972551856770},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663257672681,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.125103016+08:00","start-ts":457114667019403266,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668264587271} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114672026353675,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/491edc74-0e51-4ac6-9569-c8c1fbc04b15 {"id":"491edc74-0e51-4ac6-9569-c8c1fbc04b15","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754105} /tidb/cdc/default/__cdc_meta__/capture/9ecef71c-21ca-4c23-b07f-8f7fe1caeaab {"id":"9ecef71c-21ca-4c23-b07f-8f7fe1caeaab","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754079} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd75a1086 491edc74-0e51-4ac6-9569-c8c1fbc04b15 /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a0bc5 9ecef71c-21ca-4c23-b07f-8f7fe1caeaab /tidb/cdc/default/__cdc_meta__/owner/223195ffd75a1086 491edc74-0e51-4ac6-9569-c8c1fbc04b15 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17100088867030923722,"high":15578063972551856770},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663257672681,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.125103016+08:00","start-ts":457114667019403266,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668264587271} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114672026353675,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7ac1db7f-9083-451a-8065-dae912bbbd5a {"id":"7ac1db7f-9083-451a-8065-dae912bbbd5a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754105} /tidb/cdc/default/__cdc_meta__/capture/ad029377-c647-4ed5-9b3a-5a8749f93c6a {"id":"ad029377-c647-4ed5-9b3a-5a8749f93c6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754102} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50926 7ac1db7f-9083-451a-8065-dae912bbbd5a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50926 7ac1db7f-9083-451a-8065-dae912bbbd5a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114674606637084,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7ac1db7f-9083-451a-8065-dae912bbbd5a {"id":"7ac1db7f-9083-451a-8065-dae912bbbd5a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754105} /tidb/cdc/default/__cdc_meta__/capture/ad029377-c647-4ed5-9b3a-5a8749f93c6a {"id":"ad029377-c647-4ed5-9b3a-5a8749f93c6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754102} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50926 7ac1db7f-9083-451a-8065-dae912bbbd5a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50926 7ac1db7f-9083-451a-8065-dae912bbbd5a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114674606637084,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7ac1db7f-9083-451a-8065-dae912bbbd5a {"id":"7ac1db7f-9083-451a-8065-dae912bbbd5a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754105} /tidb/cdc/default/__cdc_meta__/capture/ad029377-c647-4ed5-9b3a-5a8749f93c6a {"id":"ad029377-c647-4ed5-9b3a-5a8749f93c6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754102} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b507e4 c65255a7-4e05-470b-8f82-9e32b067a6b6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50926 7ac1db7f-9083-451a-8065-dae912bbbd5a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50926 7ac1db7f-9083-451a-8065-dae912bbbd5a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114674606637084,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later [Fri Apr 4 16:08:28 CST 2025] <<<<<< START cdc server in batch_update_to_no_batch case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.77627764.out server --log-file /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later command: check_changefeed_state http://127.0.0.1:2379 1364080071926037623110010353592831602866 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=1364080071926037623110010353592831602866 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c 1364080071926037623110010353592831602866 -s ++ grep -v 'Command to ticdc' Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... + info='{ "upstream_id": 7489366707262181732, "id": "1364080071926037623110010353592831602866", "namespace": "default", "state": "normal", "checkpoint_tso": 457114676740227075, "checkpoint_time": "2025-04-04 16:08:27.438", "error": null }' + echo '{ "upstream_id": 7489366707262181732, "id": "1364080071926037623110010353592831602866", "namespace": "default", "state": "normal", "checkpoint_tso": 457114676740227075, "checkpoint_time": "2025-04-04 16:08:27.438", "error": null }' { "upstream_id": 7489366707262181732, "id": "1364080071926037623110010353592831602866", "namespace": "default", "state": "normal", "checkpoint_tso": 457114676740227075, "checkpoint_time": "2025-04-04 16:08:27.438", "error": null } ++ echo '{' '"upstream_id":' 7489366707262181732, '"id":' '"1364080071926037623110010353592831602866",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114676740227075, '"checkpoint_time":' '"2025-04-04' '16:08:27.438",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366707262181732, '"id":' '"1364080071926037623110010353592831602866",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114676740227075, '"checkpoint_time":' '"2025-04-04' '16:08:27.438",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test1.table1 not exists for 1-th check, retry later command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_J Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8321) - No such process wait process 8321 exit for 1-th time... process 8321 already exit + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:30 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cc804a6-0f3d-40da-a221-0319cac40e7b {"id":"9cc804a6-0f3d-40da-a221-0319cac40e7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754107} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cc804a6-0f3d-40da-a221-0319cac40e7b {"id":"9cc804a6-0f3d-40da-a221-0319cac40e7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754107} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cc804a6-0f3d-40da-a221-0319cac40e7b {"id":"9cc804a6-0f3d-40da-a221-0319cac40e7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754107} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114676377419777 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.cli.8291.out cli changefeed create --start-ts=457114676377419777 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489366817178628491,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:30.808709183+08:00","start_ts":457114676377419777,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114676377419777,"checkpoint_ts":457114676377419777,"checkpoint_time":"2025-04-04 16:08:26.054","gid":{"low":13276928321903842334,"high":16507343611319947687}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later check diff successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8399) - No such process wait process 8399 exit for 1-th time... process 8399 already exit [Fri Apr 4 16:08:31 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.84828484.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test1.table1 exists table test1.table2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10457) - No such process wait process 10457 exit for 1-th time... process 10457 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + set +x Command succeeded on attempt 1 check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:31 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e258136-5a98-4f52-8eda-b02248a7b964 {"id":"3e258136-5a98-4f52-8eda-b02248a7b964","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754109} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e851e9 3e258136-5a98-4f52-8eda-b02248a7b964 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e851e9 3e258136-5a98-4f52-8eda-b02248a7b964' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e258136-5a98-4f52-8eda-b02248a7b964 {"id":"3e258136-5a98-4f52-8eda-b02248a7b964","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754109} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e851e9 3e258136-5a98-4f52-8eda-b02248a7b964 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e851e9 + grep -q 'failed to get info:' 3e258136-5a98-4f52-8eda-b02248a7b964' + grep -q 'etcd info' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e258136-5a98-4f52-8eda-b02248a7b964 {"id":"3e258136-5a98-4f52-8eda-b02248a7b964","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754109} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e851e9 3e258136-5a98-4f52-8eda-b02248a7b964 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e851e9 3e258136-5a98-4f52-8eda-b02248a7b964' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.7827.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489366821928375898,"id":"test","namespace":"default","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2025-04-04T16:08:32.361322521+08:00","start_ts":457114678022111235,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114678022111235,"checkpoint_ts":457114678022111235,"checkpoint_time":"2025-04-04 16:08:32.328","gid":{"low":3766683603407697203,"high":14650878310915156877}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 91000 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 Verifying downstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/stdout.log no DATA RACE found [Fri Apr 4 16:08:33 CST 2025] <<<<<< run test case changefeed_resume_with_checkpoint_ts success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 4-th time, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366850812683612 TiDB Cluster ID. tikv_gc_leader_uuid 657ff60c18c0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:4861, start at 2025-04-04 16:08:33.124102458 +0800 CST m=+3.536319466 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:33.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:33.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:33.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366850812683612 TiDB Cluster ID. tikv_gc_leader_uuid 657ff60c18c0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:4861, start at 2025-04-04 16:08:33.124102458 +0800 CST m=+3.536319466 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:33.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:33.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:33.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366860044311393 TiDB Cluster ID. tikv_gc_leader_uuid 657ff60bffc0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:4916, start at 2025-04-04 16:08:33.036133416 +0800 CST m=+3.302696119 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:33.044 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:33.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:33.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... table fail_over_ddl_test.test2 not exists for 1-th check, retry later TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366861311330212 TiDB Cluster ID. tikv_gc_leader_uuid 657ff60bd700016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:7024, start at 2025-04-04 16:08:32.892575718 +0800 CST m=+3.168946248 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:32.899 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:32.960 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:32.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366861311330212 TiDB Cluster ID. tikv_gc_leader_uuid 657ff60bd700016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:7024, start at 2025-04-04 16:08:32.892575718 +0800 CST m=+3.168946248 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:32.899 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:32.960 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:32.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366856503241784 TiDB Cluster ID. tikv_gc_leader_uuid 657ff60c53c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:7096, start at 2025-04-04 16:08:33.372478855 +0800 CST m=+3.500651123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:33.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:33.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:33.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:34 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c622942-68eb-48a0-8419-7acef4d21e16 {"id":"8c622942-68eb-48a0-8419-7acef4d21e16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754111} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114676900134937,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c622942-68eb-48a0-8419-7acef4d21e16 {"id":"8c622942-68eb-48a0-8419-7acef4d21e16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754111} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114676900134937,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c622942-68eb-48a0-8419-7acef4d21e16 {"id":"8c622942-68eb-48a0-8419-7acef4d21e16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754111} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114676900134937,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":1}' + echo '{"count":1}' {"count":1} ++ echo '{"count":1}' ++ jq -r .count + value=1 + '[' -1 == -1 ']' + '[' 1 '!=' null ']' + exit 0 [Fri Apr 4 16:08:34 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.85468548.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 91000 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=3 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 + run_sql 'SELECT * from test.t1' 127.0.0.1 3306 + check_contains 'b: 123456' + check_contains 'b: aaaaa' + check_contains 'c: ab\\\\c' + check_contains 'c: ab\\c' + ret=0 + '[' 0 == 0 ']' + echo 'check data successfully' check data successfully + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 + '[' mysql == mysql ']' + check_table_exists test.t2 127.0.0.1 3306 table test.t2 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseC-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table fail_over_ddl_test.test2 exists Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8229) - No such process wait process 8229 exit for 1-th time... process 8229 already exit [Fri Apr 4 16:08:36 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.83618363.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.6264.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.8357.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":1}' + echo '{"count":1}' {"count":1} ++ echo '{"count":1}' ++ jq -r .count + value=1 + '[' -1 == -1 ']' + '[' 1 '!=' null ']' + exit 0 [Fri Apr 4 16:08:36 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1073210734.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c622942-68eb-48a0-8419-7acef4d21e16 {"id":"8c622942-68eb-48a0-8419-7acef4d21e16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754111} /tidb/cdc/default/__cdc_meta__/capture/e64ced68-cda1-4fcf-a7a1-c40003b9ba4e {"id":"e64ced68-cda1-4fcf-a7a1-c40003b9ba4e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754114} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50aa6 e64ced68-cda1-4fcf-a7a1-c40003b9ba4e /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50aa6 e64ced68-cda1-4fcf-a7a1-c40003b9ba4e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114678525427720,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c622942-68eb-48a0-8419-7acef4d21e16 {"id":"8c622942-68eb-48a0-8419-7acef4d21e16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754111} /tidb/cdc/default/__cdc_meta__/capture/e64ced68-cda1-4fcf-a7a1-c40003b9ba4e {"id":"e64ced68-cda1-4fcf-a7a1-c40003b9ba4e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754114} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50aa6 e64ced68-cda1-4fcf-a7a1-c40003b9ba4e /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50aa6 e64ced68-cda1-4fcf-a7a1-c40003b9ba4e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test + grep -q 'failed to get info:' {"checkpoint-ts":457114678525427720,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c622942-68eb-48a0-8419-7acef4d21e16 {"id":"8c622942-68eb-48a0-8419-7acef4d21e16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754111} /tidb/cdc/default/__cdc_meta__/capture/e64ced68-cda1-4fcf-a7a1-c40003b9ba4e {"id":"e64ced68-cda1-4fcf-a7a1-c40003b9ba4e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754114} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b508c2 ad029377-c647-4ed5-9b3a-5a8749f93c6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7b50aa6 e64ced68-cda1-4fcf-a7a1-c40003b9ba4e /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50a44 8c622942-68eb-48a0-8419-7acef4d21e16 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7b50aa6 e64ced68-cda1-4fcf-a7a1-c40003b9ba4e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1676362822815118304,"high":13189067738902776737},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366762938365563,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:17.872395277+08:00","start-ts":457114672981082113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114674226266120} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114678525427720,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 exists + sleep 10 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114679138320385 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114679138320385 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x + set +x + tso='457114679144873985 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114679144873985 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:08:37 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.83838385.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 4 16:08:38 CST 2025] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.62976299.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.7873.out cli changefeed pause -c test PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366879933388635 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6114900008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:6385, start at 2025-04-04 16:08:38.44592473 +0800 CST m=+3.365232948 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:38.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:38.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:38.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366879933388635 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6114900008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:6385, start at 2025-04-04 16:08:38.44592473 +0800 CST m=+3.365232948 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:38.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:38.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:38.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366875900156303 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6117b00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:6464, start at 2025-04-04 16:08:38.649952196 +0800 CST m=+3.389974639 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:38.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:38.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:38.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3087 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ace133e4-5ede-4a1b-b8d9-c246e4bd592e {"id":"ace133e4-5ede-4a1b-b8d9-c246e4bd592e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754116} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114678133784599,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ace133e4-5ede-4a1b-b8d9-c246e4bd592e {"id":"ace133e4-5ede-4a1b-b8d9-c246e4bd592e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754116} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114678133784599,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ace133e4-5ede-4a1b-b8d9-c246e4bd592e {"id":"ace133e4-5ede-4a1b-b8d9-c246e4bd592e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754116} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114678133784599,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:08:39 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.84288430.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7629666d-4c4d-4121-a3f0-0d82f4ad9c16 {"id":"7629666d-4c4d-4121-a3f0-0d82f4ad9c16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754117} /tidb/cdc/default/__cdc_meta__/capture/cc0e4a6c-c0f8-4972-af62-12c677ee15b2 {"id":"cc0e4a6c-c0f8-4972-af62-12c677ee15b2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754083} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760fd23 7629666d-4c4d-4121-a3f0-0d82f4ad9c16 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760fd23 7629666d-4c4d-4121-a3f0-0d82f4ad9c16 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12919736653804788008,"high":4951989450191133113},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663460751938,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.542651828+08:00","start-ts":457114667116658689,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668374949894} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114679201759239,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7629666d-4c4d-4121-a3f0-0d82f4ad9c16 {"id":"7629666d-4c4d-4121-a3f0-0d82f4ad9c16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754117} /tidb/cdc/default/__cdc_meta__/capture/cc0e4a6c-c0f8-4972-af62-12c677ee15b2 {"id":"cc0e4a6c-c0f8-4972-af62-12c677ee15b2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754083} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760fd23 7629666d-4c4d-4121-a3f0-0d82f4ad9c16 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760fd23 7629666d-4c4d-4121-a3f0-0d82f4ad9c16 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12919736653804788008,"high":4951989450191133113},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663460751938,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.542651828+08:00","start-ts":457114667116658689,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668374949894} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114679201759239,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7629666d-4c4d-4121-a3f0-0d82f4ad9c16 {"id":"7629666d-4c4d-4121-a3f0-0d82f4ad9c16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754117} /tidb/cdc/default/__cdc_meta__/capture/cc0e4a6c-c0f8-4972-af62-12c677ee15b2 {"id":"cc0e4a6c-c0f8-4972-af62-12c677ee15b2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754083} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd760fd23 7629666d-4c4d-4121-a3f0-0d82f4ad9c16 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760f728 cc0e4a6c-c0f8-4972-af62-12c677ee15b2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd760fd23 7629666d-4c4d-4121-a3f0-0d82f4ad9c16 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12919736653804788008,"high":4951989450191133113},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366663460751938,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:07:55.542651828+08:00","start-ts":457114667116658689,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114668374949894} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114679201759239,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366879101821917 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6123e00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:8699, start at 2025-04-04 16:08:39.420764865 +0800 CST m=+3.559581544 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:39.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:39.416 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:39.416 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366879101821917 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6123e00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:8699, start at 2025-04-04 16:08:39.420764865 +0800 CST m=+3.559581544 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:39.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:39.416 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:39.416 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366885897560908 TiDB Cluster ID. tikv_gc_leader_uuid 657ff612700000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:8772, start at 2025-04-04 16:08:39.639992705 +0800 CST m=+3.618477683 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:39.648 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:39.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:39.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.7899.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=false' --no-confirm Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=true To:mysql://root@127.0.0.1:3306/?batch-dml-enable=false} Update changefeed config successfully! ID: test Info: {"upstream_id":7489366821928375898,"id":"test","namespace":"default","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=false","create_time":"2025-04-04T16:08:32.361322521+08:00","start_ts":457114678022111235,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114678651256866,"checkpoint_ts":457114678651256866,"checkpoint_time":"2025-04-04 16:08:34.728","gid":{"low":3766683603407697203,"high":14650878310915156877}} PASS coverage: 2.4% of statements in github.com/pingcap/ticdc/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:41 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d42296de-59b9-4048-ad46-958fb6c73865 {"id":"d42296de-59b9-4048-ad46-958fb6c73865","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754118} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865 /tidb/cdc/default/__cdc_meta__/owner/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d42296de-59b9-4048-ad46-958fb6c73865 {"id":"d42296de-59b9-4048-ad46-958fb6c73865","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754118} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865 /tidb/cdc/default/__cdc_meta__/owner/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d42296de-59b9-4048-ad46-958fb6c73865 {"id":"d42296de-59b9-4048-ad46-958fb6c73865","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754118} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865 /tidb/cdc/default/__cdc_meta__/owner/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114679144873985 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.8450.out cli changefeed create --start-ts=457114679144873985 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489366861311330212,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:41.37280854+08:00","start_ts":457114679144873985,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114679144873985,"checkpoint_ts":457114679144873985,"checkpoint_time":"2025-04-04 16:08:36.611","gid":{"low":12773142019420507618,"high":17286550619415136416}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 4-th time, retry later check diff failed 2-th time, retry later check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.7793.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:42 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/724b686d-f23f-4eaa-9f77-ae65cc25d603 {"id":"724b686d-f23f-4eaa-9f77-ae65cc25d603","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754119} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd80a835e 724b686d-f23f-4eaa-9f77-ae65cc25d603 /tidb/cdc/default/__cdc_meta__/owner/223195ffd80a835e 724b686d-f23f-4eaa-9f77-ae65cc25d603' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/724b686d-f23f-4eaa-9f77-ae65cc25d603 {"id":"724b686d-f23f-4eaa-9f77-ae65cc25d603","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754119} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd80a835e 724b686d-f23f-4eaa-9f77-ae65cc25d603 /tidb/cdc/default/__cdc_meta__/owner/223195ffd80a835e 724b686d-f23f-4eaa-9f77-ae65cc25d603' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/724b686d-f23f-4eaa-9f77-ae65cc25d603 {"id":"724b686d-f23f-4eaa-9f77-ae65cc25d603","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754119} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd80a835e 724b686d-f23f-4eaa-9f77-ae65cc25d603 /tidb/cdc/default/__cdc_meta__/owner/223195ffd80a835e 724b686d-f23f-4eaa-9f77-ae65cc25d603' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.6381.out cli changefeed create --start-ts=457114679138320385 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseG-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8218) - No such process wait process 8218 exit for 1-th time... process 8218 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8296) - No such process wait process 8296 exit for 1-th time... process 8296 already exit [Fri Apr 4 16:08:41 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.84118413.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.9984.out cli tso query --pd=http://127.0.0.1:2379 [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489366850812683612,"id":"custom-changefeed-name","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:42.41612379+08:00","start_ts":457114679138320385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114679138320385,"checkpoint_ts":457114679138320385,"checkpoint_time":"2025-04-04 16:08:36.586","gid":{"low":12415339866556602221,"high":1683936240524875186}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.7925.out cli changefeed resume -c test + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later PASS coverage: 2.3% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:42 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3668 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c54a9db-92f5-400e-b93e-0efeeb048320 {"id":"5c54a9db-92f5-400e-b93e-0efeeb048320","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754119} /tidb/cdc/default/__cdc_meta__/capture/ace133e4-5ede-4a1b-b8d9-c246e4bd592e {"id":"ace133e4-5ede-4a1b-b8d9-c246e4bd592e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754116} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c74f 5c54a9db-92f5-400e-b93e-0efeeb048320 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c74f 5c54a9db-92f5-400e-b93e-0efeeb048320 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114678133784599,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c54a9db-92f5-400e-b93e-0efeeb048320 {"id":"5c54a9db-92f5-400e-b93e-0efeeb048320","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754119} /tidb/cdc/default/__cdc_meta__/capture/ace133e4-5ede-4a1b-b8d9-c246e4bd592e {"id":"ace133e4-5ede-4a1b-b8d9-c246e4bd592e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754116} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c74f 5c54a9db-92f5-400e-b93e-0efeeb048320 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c74f 5c54a9db-92f5-400e-b93e-0efeeb048320 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114678133784599,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c54a9db-92f5-400e-b93e-0efeeb048320 {"id":"5c54a9db-92f5-400e-b93e-0efeeb048320","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754119} /tidb/cdc/default/__cdc_meta__/capture/ace133e4-5ede-4a1b-b8d9-c246e4bd592e {"id":"ace133e4-5ede-4a1b-b8d9-c246e4bd592e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754116} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c5e4 9cc804a6-0f3d-40da-a221-0319cac40e7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c74f 5c54a9db-92f5-400e-b93e-0efeeb048320 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1c74f 5c54a9db-92f5-400e-b93e-0efeeb048320 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114678133784599,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/autorandom/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 16:08:43.026 +08:00] [DEBUG] [pd.rs:582] ["update latency statistics for raftstore-v1"] [tick=385] [thread_id=88] [2025/04/04 16:08:43.026 +08:00] [DEBUG] [pool.rs:146] ["scheduling task inspect raftstore latency: RaftDisk"] [thread_id=88] [2025/04/04 16:08:43.026 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="inspect raftstore latency: RaftDisk"] [thread_id=30] [2025/04/04 16:08:43.026 +08:00] [DEBUG] [pool.rs:146] ["scheduling task compute slow score: id 386, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(21.115ยตs), store_process_duration: Some(7.827ยตs), store_write_duration: Some(16.752ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=80] [2025/04/04 16:08:43.027 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="compute slow score: id 386, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(21.115ยตs), store_process_duration: Some(7.827ยตs), store_write_duration: Some(16.752ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=30] + set +x + tso='457114680565694465 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114680565694465 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x + set +x table test.simple exists table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c custom-changefeed-name -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114679492214801, "checkpoint_time": "2025-04-04 16:08:37.936", "error": null }' + echo '{ "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114679492214801, "checkpoint_time": "2025-04-04 16:08:37.936", "error": null }' { "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114679492214801, "checkpoint_time": "2025-04-04 16:08:37.936", "error": null } ++ echo '{' '"upstream_id":' 7489366850812683612, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114679492214801, '"checkpoint_time":' '"2025-04-04' '16:08:37.936",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366850812683612, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114679492214801, '"checkpoint_time":' '"2025-04-04' '16:08:37.936",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_C Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x + set +x + tso='457114680765972482 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114680765972482 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:08:44 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1002710029.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 4 16:08:44 CST 2025] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.78477851.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server1 --cluster-id test1 --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseE-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:44 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd774252-a569-49f3-a487-08589273a98f {"id":"dd774252-a569-49f3-a487-08589273a98f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754121} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/__cdc_meta__/owner/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114679715823625,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd774252-a569-49f3-a487-08589273a98f {"id":"dd774252-a569-49f3-a487-08589273a98f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754121} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/__cdc_meta__/owner/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114679715823625,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd774252-a569-49f3-a487-08589273a98f {"id":"dd774252-a569-49f3-a487-08589273a98f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754121} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/__cdc_meta__/owner/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114679715823625,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":1}' + echo '{"count":1}' {"count":1} ++ echo '{"count":1}' ++ jq -r .count + value=1 + '[' -1 == -1 ']' + '[' 1 '!=' null ']' + exit 0 [Fri Apr 4 16:08:44 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.84808482.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.6646.out cli changefeed --changefeed-id custom-changefeed-name pause ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8388) - No such process wait process 8388 exit for 1-th time... process 8388 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:08:45 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.85338535.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.7966.out cli changefeed pause -c test PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.7992.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' --no-confirm + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:47 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1aa444e-e5a3-4439-9c13-4abff91be514 {"id":"c1aa444e-e5a3-4439-9c13-4abff91be514","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754124} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1aa444e-e5a3-4439-9c13-4abff91be514 {"id":"c1aa444e-e5a3-4439-9c13-4abff91be514","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754124} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1aa444e-e5a3-4439-9c13-4abff91be514 {"id":"c1aa444e-e5a3-4439-9c13-4abff91be514","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754124} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114680765972482 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.10094.out cli changefeed create --start-ts=457114680765972482 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489366879101821917,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:47.57219326+08:00","start_ts":457114680765972482,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114680765972482,"checkpoint_ts":457114680765972482,"checkpoint_time":"2025-04-04 16:08:42.795","gid":{"low":11331767040923245914,"high":3316466741721916335}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + run_sql 'SELECT * from test.t2' 127.0.0.1 3306 + check_contains 'a: 2023-02-08' + ret=0 + '[' 0 == 0 ']' + echo 'check data successfully' check data successfully + stop_tidb_cluster Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=false To:mysql://root@127.0.0.1:3306/?batch-dml-enable=true} Update changefeed config successfully! ID: test Info: {"upstream_id":7489366821928375898,"id":"test","namespace":"default","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2025-04-04T16:08:32.361322521+08:00","start_ts":457114678022111235,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114678795436047,"checkpoint_ts":457114678795436047,"checkpoint_time":"2025-04-04 16:08:35.278","gid":{"low":3766683603407697203,"high":14650878310915156877}} PASS coverage: 2.4% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:47 GMT < Content-Length: 597 < Content-Type: text/plain; charset=utf-8 < { [597 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/15d95c72-3463-4a2e-a0a5-48e2e192219a {"id":"15d95c72-3463-4a2e-a0a5-48e2e192219a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754124} /tidb/cdc/test1/__cdc_meta__/log_coordinator/223195ffd823e5fa 15d95c72-3463-4a2e-a0a5-48e2e192219a /tidb/cdc/test1/__cdc_meta__/owner/223195ffd823e5fa 15d95c72-3463-4a2e-a0a5-48e2e192219a' + echo ' *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/15d95c72-3463-4a2e-a0a5-48e2e192219a {"id":"15d95c72-3463-4a2e-a0a5-48e2e192219a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754124} /tidb/cdc/test1/__cdc_meta__/log_coordinator/223195ffd823e5fa 15d95c72-3463-4a2e-a0a5-48e2e192219a /tidb/cdc/test1/__cdc_meta__/owner/223195ffd823e5fa 15d95c72-3463-4a2e-a0a5-48e2e192219a' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/15d95c72-3463-4a2e-a0a5-48e2e192219a {"id":"15d95c72-3463-4a2e-a0a5-48e2e192219a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754124} /tidb/cdc/test1/__cdc_meta__/log_coordinator/223195ffd823e5fa 15d95c72-3463-4a2e-a0a5-48e2e192219a /tidb/cdc/test1/__cdc_meta__/owner/223195ffd823e5fa 15d95c72-3463-4a2e-a0a5-48e2e192219a' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:08:47 CST 2025] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.79057907.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server2 --cluster-id test2 --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8807e51-22dc-4c22-96ef-f0ba9ec3d94d {"id":"d8807e51-22dc-4c22-96ef-f0ba9ec3d94d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754125} /tidb/cdc/default/__cdc_meta__/capture/dd774252-a569-49f3-a487-08589273a98f {"id":"dd774252-a569-49f3-a487-08589273a98f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754121} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd789a352 d8807e51-22dc-4c22-96ef-f0ba9ec3d94d /tidb/cdc/default/__cdc_meta__/owner/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/__cdc_meta__/owner/223195ffd789a352 d8807e51-22dc-4c22-96ef-f0ba9ec3d94d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114681026281478,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8807e51-22dc-4c22-96ef-f0ba9ec3d94d {"id":"d8807e51-22dc-4c22-96ef-f0ba9ec3d94d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754125} /tidb/cdc/default/__cdc_meta__/capture/dd774252-a569-49f3-a487-08589273a98f {"id":"dd774252-a569-49f3-a487-08589273a98f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754121} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd789a352 d8807e51-22dc-4c22-96ef-f0ba9ec3d94d /tidb/cdc/default/__cdc_meta__/owner/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/__cdc_meta__/owner/223195ffd789a352 d8807e51-22dc-4c22-96ef-f0ba9ec3d94d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114681026281478,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8807e51-22dc-4c22-96ef-f0ba9ec3d94d {"id":"d8807e51-22dc-4c22-96ef-f0ba9ec3d94d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754125} /tidb/cdc/default/__cdc_meta__/capture/dd774252-a569-49f3-a487-08589273a98f {"id":"dd774252-a569-49f3-a487-08589273a98f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754121} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7899df0 458433fc-0359-4bd3-b843-6a8d50123b3c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd789a352 d8807e51-22dc-4c22-96ef-f0ba9ec3d94d /tidb/cdc/default/__cdc_meta__/owner/223195ffd789a2ee dd774252-a569-49f3-a487-08589273a98f /tidb/cdc/default/__cdc_meta__/owner/223195ffd789a352 d8807e51-22dc-4c22-96ef-f0ba9ec3d94d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4561607153064955909,"high":2610855494131874458},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366705458091614,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:08.436061797+08:00","start-ts":457114670501199881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114671759753217} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114681026281478,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:48 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ce550ea-5650-4405-895e-9698c7a1dcc5 {"id":"4ce550ea-5650-4405-895e-9698c7a1dcc5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754125} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12773142019420507618,"high":17286550619415136416},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366861311330212,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:41.37280854+08:00","start-ts":457114679144873985,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114680390057990} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114680769904646,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ce550ea-5650-4405-895e-9698c7a1dcc5 {"id":"4ce550ea-5650-4405-895e-9698c7a1dcc5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754125} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12773142019420507618,"high":17286550619415136416},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366861311330212,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:41.37280854+08:00","start-ts":457114679144873985,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114680390057990} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114680769904646,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ce550ea-5650-4405-895e-9698c7a1dcc5 {"id":"4ce550ea-5650-4405-895e-9698c7a1dcc5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754125} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12773142019420507618,"high":17286550619415136416},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366861311330212,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:41.37280854+08:00","start-ts":457114679144873985,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114680390057990} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114680769904646,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:08:48 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.86118613.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Run 8 test check diff failed 1-th time, retry later check diff failed 2-th time, retry later + set +x Command succeeded on attempt 1 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.8019.out cli changefeed resume -c test table sink_retry.finish_mark_2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/mysql_sink_retry/stdout.log no DATA RACE found [Fri Apr 4 16:08:46 CST 2025] <<<<<< run test case mysql_sink_retry success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366845632123003 TiDB Cluster ID. tikv_gc_leader_uuid 657ff61af44000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:6252, start at 2025-04-04 16:08:48.356622174 +0800 CST m=+21.694291106 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:48.363 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:48.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:48.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366845632123003 TiDB Cluster ID. tikv_gc_leader_uuid 657ff61af44000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:6252, start at 2025-04-04 16:08:48.356622174 +0800 CST m=+21.694291106 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:48.363 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:48.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:48.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366849982076857 TiDB Cluster ID. tikv_gc_leader_uuid 657ff609f4c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:6336, start at 2025-04-04 16:08:30.968672661 +0800 CST m=+4.157772531 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:30.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:30.931 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:30.931 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... PASS coverage: 2.3% of statements in github.com/pingcap/ticdc/... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_G Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c custom-changefeed-name -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114679492214801, "checkpoint_time": "2025-04-04 16:08:37.936", "error": null }' + echo '{ "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114679492214801, "checkpoint_time": "2025-04-04 16:08:37.936", "error": null }' { "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114679492214801, "checkpoint_time": "2025-04-04 16:08:37.936", "error": null } ++ echo '{' '"upstream_id":' 7489366850812683612, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"stopped",' '"checkpoint_tso":' 457114679492214801, '"checkpoint_time":' '"2025-04-04' '16:08:37.936",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489366850812683612, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"stopped",' '"checkpoint_tso":' 457114679492214801, '"checkpoint_time":' '"2025-04-04' '16:08:37.936",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.6722.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7489366850812683612,"id":"custom-changefeed-name","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:42.41612379+08:00","start_ts":457114679138320385,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":true,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114679492214801,"checkpoint_ts":457114679492214801,"checkpoint_time":"2025-04-04 16:08:37.936","gid":{"low":12415339866556602221,"high":1683936240524875186}} PASS coverage: 3.3% of statements in github.com/pingcap/ticdc/... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table fail_over_ddl_test.test2 not exists for 1-th check, retry later check diff failed 2-th time, retry later + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:50 GMT < Content-Length: 597 < Content-Type: text/plain; charset=utf-8 < { [597 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/c42457b1-26fc-4125-a06b-b2ec0e2efe13 {"id":"c42457b1-26fc-4125-a06b-b2ec0e2efe13","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754127} /tidb/cdc/test2/__cdc_meta__/log_coordinator/223195ffd823e656 c42457b1-26fc-4125-a06b-b2ec0e2efe13 /tidb/cdc/test2/__cdc_meta__/owner/223195ffd823e656 c42457b1-26fc-4125-a06b-b2ec0e2efe13' + echo ' *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/c42457b1-26fc-4125-a06b-b2ec0e2efe13 {"id":"c42457b1-26fc-4125-a06b-b2ec0e2efe13","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754127} /tidb/cdc/test2/__cdc_meta__/log_coordinator/223195ffd823e656 c42457b1-26fc-4125-a06b-b2ec0e2efe13 /tidb/cdc/test2/__cdc_meta__/owner/223195ffd823e656 c42457b1-26fc-4125-a06b-b2ec0e2efe13' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/c42457b1-26fc-4125-a06b-b2ec0e2efe13 {"id":"c42457b1-26fc-4125-a06b-b2ec0e2efe13","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754127} /tidb/cdc/test2/__cdc_meta__/log_coordinator/223195ffd823e656 c42457b1-26fc-4125-a06b-b2ec0e2efe13 /tidb/cdc/test2/__cdc_meta__/owner/223195ffd823e656 c42457b1-26fc-4125-a06b-b2ec0e2efe13' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.7970.out cli changefeed create --start-ts=457114680565694465 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/multi_cdc_cluster/conf/changefeed1.toml Create changefeed successfully! ID: 1652262210037597858117384265386302043793 Info: {"upstream_id":7489366879933388635,"id":"1652262210037597858117384265386302043793","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:50.895144517+08:00","start_ts":457114680565694465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc1"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114680565694465,"checkpoint_ts":457114680565694465,"checkpoint_time":"2025-04-04 16:08:42.031","gid":{"low":16522622100375978581,"high":17384265386302043793}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... table batch_update_to_no_batch.v1 exists table batch_update_to_no_batch.recover_and_insert exists table batch_update_to_no_batch.finish_mark not exists for 1-th check, retry later [Fri Apr 4 16:08:51 CST 2025] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.1974219744.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.6761.out cli changefeed --changefeed-id custom-changefeed-name resume ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.3% of statements in github.com/pingcap/ticdc/... check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:51 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03ac071f-0323-449f-9dcd-f3621d78b349 {"id":"03ac071f-0323-449f-9dcd-f3621d78b349","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754128} /tidb/cdc/default/__cdc_meta__/capture/4ce550ea-5650-4405-895e-9698c7a1dcc5 {"id":"4ce550ea-5650-4405-895e-9698c7a1dcc5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754125} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/__cdc_meta__/owner/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12773142019420507618,"high":17286550619415136416},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366861311330212,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:41.37280854+08:00","start-ts":457114679144873985,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114680390057990} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114682880163861,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03ac071f-0323-449f-9dcd-f3621d78b349 {"id":"03ac071f-0323-449f-9dcd-f3621d78b349","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754128} /tidb/cdc/default/__cdc_meta__/capture/4ce550ea-5650-4405-895e-9698c7a1dcc5 {"id":"4ce550ea-5650-4405-895e-9698c7a1dcc5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754125} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/__cdc_meta__/owner/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12773142019420507618,"high":17286550619415136416},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366861311330212,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:41.37280854+08:00","start-ts":457114679144873985,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114680390057990} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114682880163861,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03ac071f-0323-449f-9dcd-f3621d78b349 {"id":"03ac071f-0323-449f-9dcd-f3621d78b349","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754128} /tidb/cdc/default/__cdc_meta__/capture/4ce550ea-5650-4405-895e-9698c7a1dcc5 {"id":"4ce550ea-5650-4405-895e-9698c7a1dcc5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754125} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81112e5 d42296de-59b9-4048-ad46-958fb6c73865 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/__cdc_meta__/owner/223195ffd81113c4 4ce550ea-5650-4405-895e-9698c7a1dcc5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12773142019420507618,"high":17286550619415136416},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366861311330212,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:41.37280854+08:00","start-ts":457114679144873985,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114680390057990} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114682880163861,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.8688.out cli capture list + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.7994.out cli changefeed create --start-ts=457114680565694465 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8301 --config=/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/multi_cdc_cluster/conf/changefeed2.toml table fail_over_ddl_test.test2 exists Create changefeed successfully! ID: 1486279149441868092918407774388502602656 Info: {"upstream_id":7489366879933388635,"id":"1486279149441868092918407774388502602656","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:52.661149575+08:00","start_ts":457114680565694465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc2"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114680565694465,"checkpoint_ts":457114680565694465,"checkpoint_time":"2025-04-04 16:08:42.031","gid":{"low":14862791494418680929,"high":18407774388502602656}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_E Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10032) - No such process wait process 10032 exit for 1-th time... process 10032 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:08:53 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1017010172.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + set +x check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseF-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + ans='[ { "id": "4ce550ea-5650-4405-895e-9698c7a1dcc5", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "03ac071f-0323-449f-9dcd-f3621d78b349", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"4ce550ea-5650-4405-895e-9698c7a1dcc5",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"03ac071f-0323-449f-9dcd-f3621d78b349",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=03ac071f-0323-449f-9dcd-f3621d78b349 + '[' -z 03ac071f-0323-449f-9dcd-f3621d78b349 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 03ac071f-0323-449f-9dcd-f3621d78b349 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.8716.out cli changefeed move-table -c test -t 114 -d 03ac071f-0323-449f-9dcd-f3621d78b349 table batch_update_to_no_batch.finish_mark exists check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 + set +x table test.multi_cdc1 exists cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseD-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366944643709192 TiDB Cluster ID. tikv_gc_leader_uuid 657ff61f9240006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:6979, start at 2025-04-04 16:08:53.073576253 +0800 CST m=+3.379820723 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:53.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:53.065 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:53.065 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366944643709192 TiDB Cluster ID. tikv_gc_leader_uuid 657ff61f9240006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:6979, start at 2025-04-04 16:08:53.073576253 +0800 CST m=+3.379820723 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:53.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:53.065 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:53.065 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366950582987861 TiDB Cluster ID. tikv_gc_leader_uuid 657ff61fab4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:7053, start at 2025-04-04 16:08:53.197132123 +0800 CST m=+3.397360875 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:53.205 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:53.265 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:53.265 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.multi_cdc2 exists check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:54 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/503a4ee2-d24d-40a6-8fdd-172d641ff2a7 {"id":"503a4ee2-d24d-40a6-8fdd-172d641ff2a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754131} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd80704e3 503a4ee2-d24d-40a6-8fdd-172d641ff2a7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd80704e3 503a4ee2-d24d-40a6-8fdd-172d641ff2a7' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/503a4ee2-d24d-40a6-8fdd-172d641ff2a7 {"id":"503a4ee2-d24d-40a6-8fdd-172d641ff2a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754131} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd80704e3 503a4ee2-d24d-40a6-8fdd-172d641ff2a7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd80704e3 503a4ee2-d24d-40a6-8fdd-172d641ff2a7' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/503a4ee2-d24d-40a6-8fdd-172d641ff2a7 {"id":"503a4ee2-d24d-40a6-8fdd-172d641ff2a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754131} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd80704e3 503a4ee2-d24d-40a6-8fdd-172d641ff2a7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd80704e3 503a4ee2-d24d-40a6-8fdd-172d641ff2a7' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.20070.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 54962255258945841315930326958270868888 Info: {"upstream_id":7489366845632123003,"id":"54962255258945841315930326958270868888","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:54.577687102+08:00","start_ts":457114683844067331,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114683844067331,"checkpoint_ts":457114683844067331,"checkpoint_time":"2025-04-04 16:08:54.537","gid":{"low":5496225525894584131,"high":5930326958270868888}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366940792512338 TiDB Cluster ID. tikv_gc_leader_uuid 657ff61fbe8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:11501, start at 2025-04-04 16:08:53.279552085 +0800 CST m=+3.280632337 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:53.347 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:53.341 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:53.341 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366940792512338 TiDB Cluster ID. tikv_gc_leader_uuid 657ff61fbe8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:11501, start at 2025-04-04 16:08:53.279552085 +0800 CST m=+3.280632337 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:53.347 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:53.341 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:53.341 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366947263483040 TiDB Cluster ID. tikv_gc_leader_uuid 657ff61fef40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:11588, start at 2025-04-04 16:08:53.445187989 +0800 CST m=+3.285119992 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:53.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:53.437 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:53.437 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_I/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ***************** properties ***************** "mysql.port"="4000" "threadcount"="2" "mysql.host"="127.0.0.1" "recordcount"="1000" "table"="a1" "readallfields"="true" "operationcount"="0" "updateproportion"="0" "mysql.db"="multi_rocks" "dotransactions"="false" "insertproportion"="0" "readproportion"="0" "mysql.user"="root" "workload"="core" "requestdistribution"="uniform" "scanproportion"="0" ********************************************** cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/batch_update_to_no_batch/stdout.log no DATA RACE found [Fri Apr 4 16:08:56 CST 2025] <<<<<< run test case batch_update_to_no_batch success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d431826f-9092-4ccd-99f0-9f0d086ebd87 {"id":"d431826f-9092-4ccd-99f0-9f0d086ebd87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754133} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11331767040923245914,"high":3316466741721916335},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366879101821917,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:47.57219326+08:00","start-ts":457114680765972482,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114682011156488} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114682522337304,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d431826f-9092-4ccd-99f0-9f0d086ebd87 {"id":"d431826f-9092-4ccd-99f0-9f0d086ebd87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754133} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11331767040923245914,"high":3316466741721916335},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366879101821917,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:47.57219326+08:00","start-ts":457114680765972482,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114682011156488} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114682522337304,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d431826f-9092-4ccd-99f0-9f0d086ebd87 {"id":"d431826f-9092-4ccd-99f0-9f0d086ebd87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754133} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11331767040923245914,"high":3316466741721916335},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366879101821917,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:47.57219326+08:00","start-ts":457114680765972482,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114682011156488} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114682522337304,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:08:56 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1024510247.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c custom-changefeed-name -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114683634089986, "checkpoint_time": "2025-04-04 16:08:53.736", "error": null }' + echo '{ "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114683634089986, "checkpoint_time": "2025-04-04 16:08:53.736", "error": null }' { "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114683634089986, "checkpoint_time": "2025-04-04 16:08:53.736", "error": null } ++ echo '{' '"upstream_id":' 7489366850812683612, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114683634089986, '"checkpoint_time":' '"2025-04-04' '16:08:53.736",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366850812683612, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114683634089986, '"checkpoint_time":' '"2025-04-04' '16:08:53.736",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.6847.out cli changefeed --changefeed-id custom-changefeed-name remove Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 457114683896496130 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.3% of statements in github.com/pingcap/ticdc/... Run finished, takes 538.062593ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1869.1, Avg(us): 1044, Min(us): 738, Max(us): 14006, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.db"="multi_rocks" "readallfields"="true" "recordcount"="1000" "mysql.port"="4000" "insertproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "updateproportion"="0" "table"="a2" "readproportion"="0" "operationcount"="0" "requestdistribution"="uniform" "dotransactions"="false" "workload"="core" "mysql.user"="root" "scanproportion"="0" ********************************************** [Fri Apr 4 16:08:56 CST 2025] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.83848386.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.12862.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/multi_cdc_cluster/stdoutmult_cdc.server1.log /tmp/tidb_cdc_test/multi_cdc_cluster/stdoutmult_cdc.server2.log no DATA RACE found [Fri Apr 4 16:08:56 CST 2025] <<<<<< run test case multi_cdc_cluster success! >>>>>> Run finished, takes 579.510027ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1782.0, Avg(us): 1130, Min(us): 775, Max(us): 20359, 95th(us): 2000, 99th(us): 6000 ***************** properties ***************** "workload"="core" "operationcount"="0" "mysql.port"="4000" "mysql.user"="root" "mysql.host"="127.0.0.1" "table"="a3" "dotransactions"="false" "insertproportion"="0" "readproportion"="0" "threadcount"="2" "scanproportion"="0" "mysql.db"="multi_rocks" "updateproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "recordcount"="1000" ********************************************** chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Run finished, takes 511.765384ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 2023.9, Avg(us): 995, Min(us): 763, Max(us): 17553, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "requestdistribution"="uniform" "operationcount"="0" "updateproportion"="0" "recordcount"="1000" "threadcount"="2" "mysql.user"="root" "readproportion"="0" "scanproportion"="0" "mysql.port"="4000" "dotransactions"="false" "workload"="core" "mysql.db"="multi_rocks" "readallfields"="true" "insertproportion"="0" "mysql.host"="127.0.0.1" "table"="a4" ********************************************** + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Run finished, takes 548.084808ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1884.4, Avg(us): 1061, Min(us): 757, Max(us): 20867, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "operationcount"="0" "table"="a5" "insertproportion"="0" "mysql.user"="root" "threadcount"="2" "requestdistribution"="uniform" "mysql.db"="multi_rocks" "scanproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="1000" "workload"="core" "dotransactions"="false" "readallfields"="true" "readproportion"="0" "updateproportion"="0" "mysql.port"="4000" ********************************************** + set +x + tso='457114684474523649 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114684474523649 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:08:58 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1289212894.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8538) - No such process wait process 8538 exit for 1-th time... process 8538 already exit [2025/04/04 16:08:55.421 +08:00chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Run finished, takes 619.402273ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1669.2, Avg(us): 1206, Min(us): 746, Max(us): 34052, 95th(us): 2000, 99th(us): 2000 start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_I Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:59 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ecd6c9bf-a239-497b-aef9-6c314514e8af {"id":"ecd6c9bf-a239-497b-aef9-6c314514e8af","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754136} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd862dbd8 ecd6c9bf-a239-497b-aef9-6c314514e8af /tidb/cdc/default/__cdc_meta__/owner/223195ffd862dbd8 ecd6c9bf-a239-497b-aef9-6c314514e8af' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ecd6c9bf-a239-497b-aef9-6c314514e8af {"id":"ecd6c9bf-a239-497b-aef9-6c314514e8af","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754136} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd862dbd8 ecd6c9bf-a239-497b-aef9-6c314514e8af /tidb/cdc/default/__cdc_meta__/owner/223195ffd862dbd8 ecd6c9bf-a239-497b-aef9-6c314514e8af' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ecd6c9bf-a239-497b-aef9-6c314514e8af {"id":"ecd6c9bf-a239-497b-aef9-6c314514e8af","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754136} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd862dbd8 ecd6c9bf-a239-497b-aef9-6c314514e8af /tidb/cdc/default/__cdc_meta__/owner/223195ffd862dbd8 ecd6c9bf-a239-497b-aef9-6c314514e8af' + grep -q 'etcd info' + break + set +x table multi_rocks.finish_mark not exists for 1-th check, retry later Create changefeed successfully! ID: 6360986737035169910407006225261159045 Info: {"upstream_id":7489366944643709192,"id":"6360986737035169910407006225261159045","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:08:59.462442023+08:00","start_ts":457114685131718657,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114685131718657,"checkpoint_ts":457114685131718657,"checkpoint_time":"2025-04-04 16:08:59.449","gid":{"low":6360986737035169910,"high":407006225261159045}} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:08:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01b0c1f0-1e6f-448b-9a07-ba3a30f6972b {"id":"01b0c1f0-1e6f-448b-9a07-ba3a30f6972b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754136} /tidb/cdc/default/__cdc_meta__/capture/d431826f-9092-4ccd-99f0-9f0d086ebd87 {"id":"d431826f-9092-4ccd-99f0-9f0d086ebd87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754133} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e255 01b0c1f0-1e6f-448b-9a07-ba3a30f6972b /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e255 01b0c1f0-1e6f-448b-9a07-ba3a30f6972b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11331767040923245914,"high":3316466741721916335},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366879101821917,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:47.57219326+08:00","start-ts":457114680765972482,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114682011156488} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114682522337304,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01b0c1f0-1e6f-448b-9a07-ba3a30f6972b {"id":"01b0c1f0-1e6f-448b-9a07-ba3a30f6972b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754136} /tidb/cdc/default/__cdc_meta__/capture/d431826f-9092-4ccd-99f0-9f0d086ebd87 {"id":"d431826f-9092-4ccd-99f0-9f0d086ebd87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754133} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e255 01b0c1f0-1e6f-448b-9a07-ba3a30f6972b /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e255 01b0c1f0-1e6f-448b-9a07-ba3a30f6972b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11331767040923245914,"high":3316466741721916335},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366879101821917,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:47.57219326+08:00","start-ts":457114680765972482,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114682011156488} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114682522337304,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01b0c1f0-1e6f-448b-9a07-ba3a30f6972b {"id":"01b0c1f0-1e6f-448b-9a07-ba3a30f6972b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754136} /tidb/cdc/default/__cdc_meta__/capture/d431826f-9092-4ccd-99f0-9f0d086ebd87 {"id":"d431826f-9092-4ccd-99f0-9f0d086ebd87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754133} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e0ea c1aa444e-e5a3-4439-9c13-4abff91be514 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e255 01b0c1f0-1e6f-448b-9a07-ba3a30f6972b /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e255 01b0c1f0-1e6f-448b-9a07-ba3a30f6972b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11331767040923245914,"high":3316466741721916335},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366879101821917,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:47.57219326+08:00","start-ts":457114680765972482,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114682011156488} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114682522337304,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.10311.out cli capture list VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366969832589403 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6254b4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:9614, start at 2025-04-04 16:08:58.940699892 +0800 CST m=+3.547717422 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:58.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:58.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:58.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366969832589403 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6254b4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:9614, start at 2025-04-04 16:08:58.940699892 +0800 CST m=+3.547717422 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:58.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:58.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:58.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366964512350264 TiDB Cluster ID. tikv_gc_leader_uuid 657ff625194000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:9677, start at 2025-04-04 16:08:58.754863938 +0800 CST m=+3.212226950 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:10:58.762 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:08:58.766 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:58:58.766 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table autorandom_test.table_a not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + ans='[ { "id": "d431826f-9092-4ccd-99f0-9f0d086ebd87", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "01b0c1f0-1e6f-448b-9a07-ba3a30f6972b", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"d431826f-9092-4ccd-99f0-9f0d086ebd87",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"01b0c1f0-1e6f-448b-9a07-ba3a30f6972b",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=01b0c1f0-1e6f-448b-9a07-ba3a30f6972b + '[' -z 01b0c1f0-1e6f-448b-9a07-ba3a30f6972b ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 01b0c1f0-1e6f-448b-9a07-ba3a30f6972b + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.10340.out cli changefeed move-table -c test -t 114 -d 01b0c1f0-1e6f-448b-9a07-ba3a30f6972b changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.6908.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name table multi_rocks.finish_mark not exists for 2-th check, retry later [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489366850812683612,"id":"custom-changefeed-name","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:01.428279715+08:00","start_ts":457114685639491587,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114685639491587,"checkpoint_ts":457114685639491587,"checkpoint_time":"2025-04-04 16:09:01.386","gid":{"low":15010167595945217658,"high":571966812069066411}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:01 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87af450b-b92c-41ca-8ead-d12bfd09a6d2 {"id":"87af450b-b92c-41ca-8ead-d12bfd09a6d2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754138} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87af450b-b92c-41ca-8ead-d12bfd09a6d2 {"id":"87af450b-b92c-41ca-8ead-d12bfd09a6d2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754138} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87af450b-b92c-41ca-8ead-d12bfd09a6d2 {"id":"87af450b-b92c-41ca-8ead-d12bfd09a6d2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754138} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114684474523649 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.12946.out cli changefeed create --start-ts=457114684474523649 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489366940792512338,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:01.705716672+08:00","start_ts":457114684474523649,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114684474523649,"checkpoint_ts":457114684474523649,"checkpoint_time":"2025-04-04 16:08:56.942","gid":{"low":7152490815851699041,"high":13728212366313176759}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_F Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_D Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.cli.10875.out cli tso query --pd=http://127.0.0.1:2379 + stop_tidb_cluster + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366983529039173 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6283240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:11729, start at 2025-04-04 16:09:01.9225067 +0800 CST m=+3.587266318 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:01.989 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:01.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:01.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366983529039173 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6283240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:11729, start at 2025-04-04 16:09:01.9225067 +0800 CST m=+3.587266318 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:01.989 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:01.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:01.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489366981480778003 TiDB Cluster ID. tikv_gc_leader_uuid 657ff628630000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:11796, start at 2025-04-04 16:09:02.124838146 +0800 CST m=+3.627838255 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:02.132 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:02.092 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:02.092 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table autorandom_test.table_a exists check diff failed 1-th time, retry later + set +x Command succeeded on attempt 1 table multi_rocks.finish_mark not exists for 3-th check, retry later table fail_over_ddl_test.test2 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8366) - No such process wait process 8366 exit for 1-th time... process 8366 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8433) - No such process wait process 8433 exit for 1-th time... process 8433 already exit [Fri Apr 4 16:09:01 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.85528554.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 + set +x + tso='457114685883285505 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114685883285505 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:03 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1091210914.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3087 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/850a4ab9-edac-4117-a51b-38bc8b65a038 {"id":"850a4ab9-edac-4117-a51b-38bc8b65a038","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754141} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114685080338442,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/850a4ab9-edac-4117-a51b-38bc8b65a038 {"id":"850a4ab9-edac-4117-a51b-38bc8b65a038","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754141} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114685080338442,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/850a4ab9-edac-4117-a51b-38bc8b65a038 {"id":"850a4ab9-edac-4117-a51b-38bc8b65a038","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754141} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114685080338442,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":1}' + echo '{"count":1}' {"count":1} ++ echo '{"count":1}' ++ jq -r .count + value=1 + '[' -1 == -1 ']' + '[' 1 '!=' null ']' + exit 0 [Fri Apr 4 16:09:04 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.86248626.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table fail_over_ddl_test.test2 exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.13088.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/autorandom/stdout.log no DATA RACE found [Fri Apr 4 16:09:05 CST 2025] <<<<<< run test case autorandom success! >>>>>> Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (12897) - No such process wait process 12897 exit for 1-th time... process 12897 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:09:05 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1303913041.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table multi_rocks.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/multi_rocks/stdout.log no DATA RACE found [Fri Apr 4 16:09:06 CST 2025] <<<<<< run test case multi_rocks success! >>>>>> command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c custom-changefeed-name -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114685993385987, "checkpoint_time": "2025-04-04 16:09:02.736", "error": null }' + echo '{ "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114685993385987, "checkpoint_time": "2025-04-04 16:09:02.736", "error": null }' { "upstream_id": 7489366850812683612, "id": "custom-changefeed-name", "namespace": "default", "state": "normal", "checkpoint_tso": 457114685993385987, "checkpoint_time": "2025-04-04 16:09:02.736", "error": null } ++ echo '{' '"upstream_id":' 7489366850812683612, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114685993385987, '"checkpoint_time":' '"2025-04-04' '16:09:02.736",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489366850812683612, '"id":' '"custom-changefeed-name",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114685993385987, '"checkpoint_time":' '"2025-04-04' '16:09:02.736",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_O/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... run task successfully Start delete service gc safepoint + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.7003.out cli unsafe delete-service-gc-safepoint + set +x + tso='457114686664736770 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114686664736770 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:06 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1311713119.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... Starting Upstream TiDB... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:06 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d5d3352-2286-46a0-bd94-01f396ca3476 {"id":"1d5d3352-2286-46a0-bd94-01f396ca3476","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754144} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d5d3352-2286-46a0-bd94-01f396ca3476 {"id":"1d5d3352-2286-46a0-bd94-01f396ca3476","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754144} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d5d3352-2286-46a0-bd94-01f396ca3476 {"id":"1d5d3352-2286-46a0-bd94-01f396ca3476","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754144} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114685883285505 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.cli.10975.out cli changefeed create --start-ts=457114685883285505 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_L/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: test Info: {"upstream_id":7489366969832589403,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:07.085549337+08:00","start_ts":457114685883285505,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114685883285505,"checkpoint_ts":457114685883285505,"checkpoint_time":"2025-04-04 16:09:02.316","gid":{"low":10468490312239419612,"high":6752150446206420402}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:07 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3668 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49e18d94-3f88-4df7-b209-8882e0d4b969 {"id":"49e18d94-3f88-4df7-b209-8882e0d4b969","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754144} /tidb/cdc/default/__cdc_meta__/capture/850a4ab9-edac-4117-a51b-38bc8b65a038 {"id":"850a4ab9-edac-4117-a51b-38bc8b65a038","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754141} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1cbc2 49e18d94-3f88-4df7-b209-8882e0d4b969 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1cbc2 49e18d94-3f88-4df7-b209-8882e0d4b969 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114686129176583,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49e18d94-3f88-4df7-b209-8882e0d4b969 {"id":"49e18d94-3f88-4df7-b209-8882e0d4b969","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754144} /tidb/cdc/default/__cdc_meta__/capture/850a4ab9-edac-4117-a51b-38bc8b65a038 {"id":"850a4ab9-edac-4117-a51b-38bc8b65a038","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754141} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1cbc2 49e18d94-3f88-4df7-b209-8882e0d4b969 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1cbc2 49e18d94-3f88-4df7-b209-8882e0d4b969 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114686129176583,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49e18d94-3f88-4df7-b209-8882e0d4b969 {"id":"49e18d94-3f88-4df7-b209-8882e0d4b969","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754144} /tidb/cdc/default/__cdc_meta__/capture/850a4ab9-edac-4117-a51b-38bc8b65a038 {"id":"850a4ab9-edac-4117-a51b-38bc8b65a038","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754141} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1c6ec ace133e4-5ede-4a1b-b8d9-c246e4bd592e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd7e1cbc2 49e18d94-3f88-4df7-b209-8882e0d4b969 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1cb60 850a4ab9-edac-4117-a51b-38bc8b65a038 /tidb/cdc/default/__cdc_meta__/owner/223195ffd7e1cbc2 49e18d94-3f88-4df7-b209-8882e0d4b969 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13276928321903842334,"high":16507343611319947687},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366817178628491,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:30.808709183+08:00","start-ts":457114676377419777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114677622341637} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114686129176583,"progress":1}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run 9 test + set +x Pass delete service gc safepoint Start reset + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.7035.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 + set +x Command succeeded on attempt 1 reset and all metadata truncated in PD! PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4a22b5f-9106-4b96-8fca-f25b40cf0948 {"id":"a4a22b5f-9106-4b96-8fca-f25b40cf0948","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754146} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7152490815851699041,"high":13728212366313176759},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366940792512338,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:01.705716672+08:00","start-ts":457114684474523649,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114685719707653} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114686099554324,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4a22b5f-9106-4b96-8fca-f25b40cf0948 {"id":"a4a22b5f-9106-4b96-8fca-f25b40cf0948","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754146} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7152490815851699041,"high":13728212366313176759},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366940792512338,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:01.705716672+08:00","start-ts":457114684474523649,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114685719707653} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114686099554324,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4a22b5f-9106-4b96-8fca-f25b40cf0948 {"id":"a4a22b5f-9106-4b96-8fca-f25b40cf0948","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754146} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7152490815851699041,"high":13728212366313176759},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366940792512338,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:01.705716672+08:00","start-ts":457114684474523649,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114685719707653} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114686099554324,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:09:08 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1311313115.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_O Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367015043551002 TiDB Cluster ID. tikv_gc_leader_uuid 657ff62e9180004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:8955, start at 2025-04-04 16:09:08.426492247 +0800 CST m=+3.163010744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:08.434 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:08.473 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:08.473 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367015043551002 TiDB Cluster ID. tikv_gc_leader_uuid 657ff62e9180004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:8955, start at 2025-04-04 16:09:08.426492247 +0800 CST m=+3.163010744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:08.434 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:08.473 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:08.473 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367005102856536 TiDB Cluster ID. tikv_gc_leader_uuid 657ff62ea500009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:9020, start at 2025-04-04 16:09:08.51527018 +0800 CST m=+3.094958814 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:08.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:08.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:08.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:09 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/832e2b7e-2343-43de-90aa-36ec4428fe9b {"id":"832e2b7e-2343-43de-90aa-36ec4428fe9b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754147} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/832e2b7e-2343-43de-90aa-36ec4428fe9b {"id":"832e2b7e-2343-43de-90aa-36ec4428fe9b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754147} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/832e2b7e-2343-43de-90aa-36ec4428fe9b {"id":"832e2b7e-2343-43de-90aa-36ec4428fe9b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754147} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114686664736770 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.13183.out cli changefeed create --start-ts=457114686664736770 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + set +x Pass reset command: ! ps -p 6302 > /dev/null 2>&1 run task failed 1-th time, retry later Create changefeed successfully! ID: test Info: {"upstream_id":7489366983529039173,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:10.0536995+08:00","start_ts":457114686664736770,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114686664736770,"checkpoint_ts":457114686664736770,"checkpoint_time":"2025-04-04 16:09:05.297","gid":{"low":4490057613477854989,"high":15543667930793430949}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_L Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table fail_over_ddl_test.test2 not exists for 1-th check, retry later check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367017944090367 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6305ac0038 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:11570, start at 2025-04-04 16:09:10.269871268 +0800 CST m=+3.280082241 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:10.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:10.332 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:10.332 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367017944090367 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6305ac0038 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:11570, start at 2025-04-04 16:09:10.269871268 +0800 CST m=+3.280082241 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:10.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:10.332 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:10.332 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367011791424810 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6308c0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:11636, start at 2025-04-04 16:09:10.478916092 +0800 CST m=+3.322945870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:10.487 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:10.499 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:10.499 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.10313.out cli tso query --pd=http://127.0.0.1:2379 command: ! ps -p 6302 > /dev/null 2>&1 run task successfully [Fri Apr 4 16:09:11 CST 2025] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.70677069.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc_cli_with_auth_restart.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data_cli_with_auth_restart --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367024372905433 TiDB Cluster ID. tikv_gc_leader_uuid 657ff630ef00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:9412, start at 2025-04-04 16:09:10.88746417 +0800 CST m=+3.438118037 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:10.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:10.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:10.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367024372905433 TiDB Cluster ID. tikv_gc_leader_uuid 657ff630ef00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:9412, start at 2025-04-04 16:09:10.88746417 +0800 CST m=+3.438118037 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:10.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:10.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:10.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367024050570627 TiDB Cluster ID. tikv_gc_leader_uuid 657ff630f04000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:9467, start at 2025-04-04 16:09:10.876840799 +0800 CST m=+3.295117301 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:10.886 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:10.899 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:10.899 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03e5bb74-7f43-42bd-8533-275906b197f5 {"id":"03e5bb74-7f43-42bd-8533-275906b197f5","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754149} /tidb/cdc/default/__cdc_meta__/capture/a4a22b5f-9106-4b96-8fca-f25b40cf0948 {"id":"a4a22b5f-9106-4b96-8fca-f25b40cf0948","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754146} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3d28 03e5bb74-7f43-42bd-8533-275906b197f5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3d28 03e5bb74-7f43-42bd-8533-275906b197f5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7152490815851699041,"high":13728212366313176759},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366940792512338,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:01.705716672+08:00","start-ts":457114684474523649,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114685719707653} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114686099554324,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03e5bb74-7f43-42bd-8533-275906b197f5 {"id":"03e5bb74-7f43-42bd-8533-275906b197f5","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754149} /tidb/cdc/default/__cdc_meta__/capture/a4a22b5f-9106-4b96-8fca-f25b40cf0948 {"id":"a4a22b5f-9106-4b96-8fca-f25b40cf0948","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754146} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3d28 03e5bb74-7f43-42bd-8533-275906b197f5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3d28 03e5bb74-7f43-42bd-8533-275906b197f5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7152490815851699041,"high":13728212366313176759},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366940792512338,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:01.705716672+08:00","start-ts":457114684474523649,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114685719707653} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114686099554324,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03e5bb74-7f43-42bd-8533-275906b197f5 {"id":"03e5bb74-7f43-42bd-8533-275906b197f5","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754149} /tidb/cdc/default/__cdc_meta__/capture/a4a22b5f-9106-4b96-8fca-f25b40cf0948 {"id":"a4a22b5f-9106-4b96-8fca-f25b40cf0948","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754146} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3beb 87af450b-b92c-41ca-8ead-d12bfd09a6d2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3d28 03e5bb74-7f43-42bd-8533-275906b197f5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3d28 03e5bb74-7f43-42bd-8533-275906b197f5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7152490815851699041,"high":13728212366313176759},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366940792512338,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:01.705716672+08:00","start-ts":457114684474523649,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114685719707653} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114686099554324,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.13189.out cli capture list table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10917) - No such process wait process 10917 exit for 1-th time... process 10917 already exit [Fri Apr 4 16:09:12 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1104311045.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 3-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseK-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + ans='[ { "id": "a4a22b5f-9106-4b96-8fca-f25b40cf0948", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "03e5bb74-7f43-42bd-8533-275906b197f5", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"a4a22b5f-9106-4b96-8fca-f25b40cf0948",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"03e5bb74-7f43-42bd-8533-275906b197f5",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=03e5bb74-7f43-42bd-8533-275906b197f5 + '[' -z 03e5bb74-7f43-42bd-8533-275906b197f5 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 03e5bb74-7f43-42bd-8533-275906b197f5 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.13222.out cli changefeed move-table -c test -t 114 -d 03e5bb74-7f43-42bd-8533-275906b197f5 + set +x + tso='457114688467238913 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114688467238913 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:13 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1034110343.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.12857.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.cli.10800.out cli tso query --pd=http://127.0.0.1:2379 table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13122) - No such process wait process 13122 exit for 1-th time... process 13122 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:09:14 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1325713259.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:14 GMT < Content-Length: 722 < Content-Type: text/plain; charset=utf-8 < { [722 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e67763f6-e3a9-4c48-8a97-0a1a25051c5c {"id":"e67763f6-e3a9-4c48-8a97-0a1a25051c5c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754152} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd80a856c e67763f6-e3a9-4c48-8a97-0a1a25051c5c /tidb/cdc/default/__cdc_meta__/owner/223195ffd80a856c e67763f6-e3a9-4c48-8a97-0a1a25051c5c /tidb/cdc/default/default/changefeed/status/custom-changefeed-name {"checkpoint-ts":457114687553142785,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e67763f6-e3a9-4c48-8a97-0a1a25051c5c {"id":"e67763f6-e3a9-4c48-8a97-0a1a25051c5c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754152} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd80a856c e67763f6-e3a9-4c48-8a97-0a1a25051c5c /tidb/cdc/default/__cdc_meta__/owner/223195ffd80a856c e67763f6-e3a9-4c48-8a97-0a1a25051c5c /tidb/cdc/default/default/changefeed/status/custom-changefeed-name {"checkpoint-ts":457114687553142785,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e67763f6-e3a9-4c48-8a97-0a1a25051c5c {"id":"e67763f6-e3a9-4c48-8a97-0a1a25051c5c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754152} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd80a856c e67763f6-e3a9-4c48-8a97-0a1a25051c5c /tidb/cdc/default/__cdc_meta__/owner/223195ffd80a856c e67763f6-e3a9-4c48-8a97-0a1a25051c5c /tidb/cdc/default/default/changefeed/status/custom-changefeed-name {"checkpoint-ts":457114687553142785,"progress":1}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.7159.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114688933593091 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114688933593091 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:15 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1289212894.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:15 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3521e32-e046-44a0-979a-6ea157515e6a {"id":"d3521e32-e046-44a0-979a-6ea157515e6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754152} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114687639388177,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3521e32-e046-44a0-979a-6ea157515e6a {"id":"d3521e32-e046-44a0-979a-6ea157515e6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754152} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114687639388177,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3521e32-e046-44a0-979a-6ea157515e6a {"id":"d3521e32-e046-44a0-979a-6ea157515e6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754152} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114687639388177,"progress":1}' + break + set +x [Fri Apr 4 16:09:15 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1110311105.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114689048674305 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114689048674305 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:15 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1082610828.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_M/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:16 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 {"id":"2dd45c5b-31a4-4e63-83bd-fb2df70b11a4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754153} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 /tidb/cdc/default/__cdc_meta__/owner/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 {"id":"2dd45c5b-31a4-4e63-83bd-fb2df70b11a4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754153} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 /tidb/cdc/default/__cdc_meta__/owner/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 {"id":"2dd45c5b-31a4-4e63-83bd-fb2df70b11a4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754153} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 /tidb/cdc/default/__cdc_meta__/owner/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114688467238913 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.10404.out cli changefeed create --start-ts=457114688467238913 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367015043551002,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:16.943277028+08:00","start_ts":457114688467238913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114688467238913,"checkpoint_ts":457114688467238913,"checkpoint_time":"2025-04-04 16:09:12.173","gid":{"low":813933754614727390,"high":2004519558964326820}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114689296400389 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114689296400389 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.7215.out cli unsafe resolve-lock --region=4 PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3084 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a169de54-5807-431c-ac52-d23fd481291e {"id":"a169de54-5807-431c-ac52-d23fd481291e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754154} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4490057613477854989,"high":15543667930793430949},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366983529039173,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:10.0536995+08:00","start-ts":457114686664736770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687909920773} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114688263553045,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a169de54-5807-431c-ac52-d23fd481291e {"id":"a169de54-5807-431c-ac52-d23fd481291e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754154} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4490057613477854989,"high":15543667930793430949},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366983529039173,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:10.0536995+08:00","start-ts":457114686664736770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687909920773} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114688263553045,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a169de54-5807-431c-ac52-d23fd481291e {"id":"a169de54-5807-431c-ac52-d23fd481291e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754154} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4490057613477854989,"high":15543667930793430949},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366983529039173,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:10.0536995+08:00","start-ts":457114686664736770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687909920773} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114688263553045,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:09:17 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1333413336.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367050049368721 TiDB Cluster ID. tikv_gc_leader_uuid 657ff637168002d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:9059, start at 2025-04-04 16:09:17.153776438 +0800 CST m=+3.766874850 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:17.163 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:17.226 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:17.226 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367050049368721 TiDB Cluster ID. tikv_gc_leader_uuid 657ff637168002d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:9059, start at 2025-04-04 16:09:17.153776438 +0800 CST m=+3.766874850 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:17.163 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:17.226 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:17.226 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367046978764669 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6371740007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:9121, start at 2025-04-04 16:09:17.159689057 +0800 CST m=+3.607896019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:17.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:17.226 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:17.226 +0800 All versions after safe point can be accessed. (DO NOT EDIT) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x Command succeeded on attempt 1 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10250) - No such process wait process 10250 exit for 1-th time... process 10250 already exit [Fri Apr 4 16:09:16 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1042510427.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.7238.out cli unsafe resolve-lock --region=4 --ts=457114689296400389 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:18 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a {"id":"68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754155} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a {"id":"68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754155} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a {"id":"68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754155} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114688933593091 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.12954.out cli changefeed create --start-ts=457114688933593091 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later PASS Create changefeed successfully! ID: test Info: {"upstream_id":7489367017944090367,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:18.712410265+08:00","start_ts":457114688933593091,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114688933593091,"checkpoint_ts":457114688933593091,"checkpoint_time":"2025-04-04 16:09:13.952","gid":{"low":4559205366755546684,"high":386323633450708134}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... coverage: 2.2% of statements in github.com/pingcap/ticdc/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367053141610012 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6384000007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:9017, start at 2025-04-04 16:09:18.345267501 +0800 CST m=+3.430205033 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:18.354 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:18.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:18.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46aaf55a-d22e-4a4b-81cc-137cc90088f1 {"id":"46aaf55a-d22e-4a4b-81cc-137cc90088f1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754156} /tidb/cdc/default/__cdc_meta__/capture/d3521e32-e046-44a0-979a-6ea157515e6a {"id":"d3521e32-e046-44a0-979a-6ea157515e6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754152} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114687639388177,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46aaf55a-d22e-4a4b-81cc-137cc90088f1 {"id":"46aaf55a-d22e-4a4b-81cc-137cc90088f1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754156} /tidb/cdc/default/__cdc_meta__/capture/d3521e32-e046-44a0-979a-6ea157515e6a {"id":"d3521e32-e046-44a0-979a-6ea157515e6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754152} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114687639388177,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46aaf55a-d22e-4a4b-81cc-137cc90088f1 {"id":"46aaf55a-d22e-4a4b-81cc-137cc90088f1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754156} /tidb/cdc/default/__cdc_meta__/capture/d3521e32-e046-44a0-979a-6ea157515e6a {"id":"d3521e32-e046-44a0-979a-6ea157515e6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754152} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8715fe5 1d5d3352-2286-46a0-bd94-01f396ca3476 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd87160e9 d3521e32-e046-44a0-979a-6ea157515e6a /tidb/cdc/default/__cdc_meta__/owner/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114687639388177,"progress":1}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:18 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbb4eddc-2e15-4280-83de-de1d42d8ded1 {"id":"dbb4eddc-2e15-4280-83de-de1d42d8ded1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754156} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbb4eddc-2e15-4280-83de-de1d42d8ded1 {"id":"dbb4eddc-2e15-4280-83de-de1d42d8ded1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754156} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbb4eddc-2e15-4280-83de-de1d42d8ded1 {"id":"dbb4eddc-2e15-4280-83de-de1d42d8ded1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754156} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114689048674305 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.cli.10886.out cli changefeed create --start-ts=457114689048674305 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367024372905433,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:19.178711511+08:00","start_ts":457114689048674305,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114689048674305,"checkpoint_ts":457114689048674305,"checkpoint_time":"2025-04-04 16:09:14.391","gid":{"low":6002285263978123182,"high":14315543091948369078}} PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367053141610012 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6384000007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:9017, start at 2025-04-04 16:09:18.345267501 +0800 CST m=+3.430205033 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:18.354 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:18.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:18.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367049356457804 TiDB Cluster ID. tikv_gc_leader_uuid 657ff638590000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:9095, start at 2025-04-04 16:09:18.456182909 +0800 CST m=+3.396948952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:18.465 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:18.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:18.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later coverage: 2.7% of statements in github.com/pingcap/ticdc/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367051100265341 TiDB Cluster ID. tikv_gc_leader_uuid 657ff638300000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:9099, start at 2025-04-04 16:09:18.285516244 +0800 CST m=+3.423978106 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:18.350 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:18.344 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:18.344 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367051100265341 TiDB Cluster ID. tikv_gc_leader_uuid 657ff638300000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:9099, start at 2025-04-04 16:09:18.285516244 +0800 CST m=+3.423978106 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:18.350 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:18.344 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:18.344 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367051504134618 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6386240007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:9155, start at 2025-04-04 16:09:18.484050759 +0800 CST m=+3.509105500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:18.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:18.544 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:18.544 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:19 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fb2a654-1daa-45a5-8261-373ef83eee93 {"id":"5fb2a654-1daa-45a5-8261-373ef83eee93","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754157} /tidb/cdc/default/__cdc_meta__/capture/d431826f-9092-4ccd-99f0-9f0d086ebd87 {"id":"d431826f-9092-4ccd-99f0-9f0d086ebd87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754133} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e640 5fb2a654-1daa-45a5-8261-373ef83eee93 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e640 5fb2a654-1daa-45a5-8261-373ef83eee93 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11331767040923245914,"high":3316466741721916335},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366879101821917,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:47.57219326+08:00","start-ts":457114680765972482,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114682011156488} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114689456046092,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fb2a654-1daa-45a5-8261-373ef83eee93 {"id":"5fb2a654-1daa-45a5-8261-373ef83eee93","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754157} /tidb/cdc/default/__cdc_meta__/capture/d431826f-9092-4ccd-99f0-9f0d086ebd87 {"id":"d431826f-9092-4ccd-99f0-9f0d086ebd87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754133} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e640 5fb2a654-1daa-45a5-8261-373ef83eee93 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e640 5fb2a654-1daa-45a5-8261-373ef83eee93 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11331767040923245914,"high":3316466741721916335},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366879101821917,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:47.57219326+08:00","start-ts":457114680765972482,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114682011156488} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114689456046092,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fb2a654-1daa-45a5-8261-373ef83eee93 {"id":"5fb2a654-1daa-45a5-8261-373ef83eee93","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754157} /tidb/cdc/default/__cdc_meta__/capture/d431826f-9092-4ccd-99f0-9f0d086ebd87 {"id":"d431826f-9092-4ccd-99f0-9f0d086ebd87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754133} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd826e640 5fb2a654-1daa-45a5-8261-373ef83eee93 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e1ed d431826f-9092-4ccd-99f0-9f0d086ebd87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd826e640 5fb2a654-1daa-45a5-8261-373ef83eee93 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11331767040923245914,"high":3316466741721916335},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366879101821917,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:47.57219326+08:00","start-ts":457114680765972482,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114682011156488} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114689456046092,"progress":1}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_M Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table fail_over_ddl_test.test2 not exists for 1-th check, retry later + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + set +x [Fri Apr 4 16:09:20 CST 2025] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.1032610328.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3665 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a169de54-5807-431c-ac52-d23fd481291e {"id":"a169de54-5807-431c-ac52-d23fd481291e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754154} /tidb/cdc/default/__cdc_meta__/capture/c84b6c0b-7b6e-413a-8736-9b7a58b23cbc {"id":"c84b6c0b-7b6e-413a-8736-9b7a58b23cbc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754157} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4490057613477854989,"high":15543667930793430949},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366983529039173,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:10.0536995+08:00","start-ts":457114686664736770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687909920773} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114688290029581,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a169de54-5807-431c-ac52-d23fd481291e {"id":"a169de54-5807-431c-ac52-d23fd481291e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754154} /tidb/cdc/default/__cdc_meta__/capture/c84b6c0b-7b6e-413a-8736-9b7a58b23cbc {"id":"c84b6c0b-7b6e-413a-8736-9b7a58b23cbc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754157} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4490057613477854989,"high":15543667930793430949},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366983529039173,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:10.0536995+08:00","start-ts":457114686664736770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687909920773} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114688290029581,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a169de54-5807-431c-ac52-d23fd481291e {"id":"a169de54-5807-431c-ac52-d23fd481291e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754154} /tidb/cdc/default/__cdc_meta__/capture/c84b6c0b-7b6e-413a-8736-9b7a58b23cbc {"id":"c84b6c0b-7b6e-413a-8736-9b7a58b23cbc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754157} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd2e3 832e2b7e-2343-43de-90aa-36ec4428fe9b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd3c3 a169de54-5807-431c-ac52-d23fd481291e /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4490057613477854989,"high":15543667930793430949},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366983529039173,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:10.0536995+08:00","start-ts":457114686664736770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687909920773} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114688290029581,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.13395.out cli capture list + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later Verifying downstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_K Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully Verifying downstream PD is started... check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (11048) - No such process wait process 11048 exit for 1-th time... process 11048 already exit + set +x + ans='[ { "id": "a169de54-5807-431c-ac52-d23fd481291e", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "c84b6c0b-7b6e-413a-8736-9b7a58b23cbc", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"a169de54-5807-431c-ac52-d23fd481291e",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"c84b6c0b-7b6e-413a-8736-9b7a58b23cbc",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=c84b6c0b-7b6e-413a-8736-9b7a58b23cbc + '[' -z c84b6c0b-7b6e-413a-8736-9b7a58b23cbc ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d c84b6c0b-7b6e-413a-8736-9b7a58b23cbc + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.13426.out cli changefeed move-table -c test -t 114 -d c84b6c0b-7b6e-413a-8736-9b7a58b23cbc + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.10357.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.10359.out cli tso query --pd=http://127.0.0.1:2379 table fail_over_ddl_test.test2 exists Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (11108) - No such process wait process 11108 exit for 1-th time... process 11108 already exit [Fri Apr 4 16:09:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1121111213.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10346) - No such process wait process 10346 exit for 1-th time... process 10346 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:09:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1048210484.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 exists table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (12897) - No such process wait process 12897 exit for 1-th time... process 12897 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:09:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1302713029.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13118) - No such process wait process 13118 exit for 1-th time... process 13118 already exit [Fri Apr 4 16:09:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1328813290.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:23 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30b07806-981c-451c-9850-41d31a488ccb {"id":"30b07806-981c-451c-9850-41d31a488ccb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754160} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bf6cda 30b07806-981c-451c-9850-41d31a488ccb /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bf6cda 30b07806-981c-451c-9850-41d31a488ccb' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30b07806-981c-451c-9850-41d31a488ccb {"id":"30b07806-981c-451c-9850-41d31a488ccb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754160} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bf6cda 30b07806-981c-451c-9850-41d31a488ccb /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bf6cda 30b07806-981c-451c-9850-41d31a488ccb' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30b07806-981c-451c-9850-41d31a488ccb {"id":"30b07806-981c-451c-9850-41d31a488ccb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754160} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bf6cda 30b07806-981c-451c-9850-41d31a488ccb /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bf6cda 30b07806-981c-451c-9850-41d31a488ccb' + grep -q 'etcd info' + break + set +x + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10831) - No such process wait process 10831 exit for 1-th time... process 10831 already exit [Fri Apr 4 16:09:23 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1096310965.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='457114690996666369 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114690996666369 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:23 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1038810390.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114691000336385 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114691000336385 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:23 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1039410396.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 238k 0 --:--:-- --:--:-- --:--:-- 238k { "version": "v9.0.0-beta.1-55-g52a1009d", "git_hash": "52a1009d7bcd84b283c5443601c9b3160b35bb7c", "id": "e67763f6-e3a9-4c48-8a97-0a1a25051c5c", "cluster_id": "default", "pid": 7072, "is_owner": true, "liveness": 0 }cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/cli_with_auth/stdout.log /tmp/tidb_cdc_test/cli_with_auth/stdout_cli_with_auth_restart.log no DATA RACE found [Fri Apr 4 16:09:23 CST 2025] <<<<<< run test case cli_with_auth success! >>>>>> Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "readproportion"="0" "dotransactions"="false" "mysql.port"="4000" "threadcount"="4" "mysql.host"="127.0.0.1" "insertproportion"="0" "mysql.user"="root" "updateproportion"="0" "requestdistribution"="uniform" "workload"="core" "scanproportion"="0" "recordcount"="50" "operationcount"="0" "mysql.db"="changefeed_reconstruct" "readallfields"="true" ********************************************** Run finished, takes 20.126586ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3291.2, Avg(us): 1544, Min(us): 970, Max(us): 5555, 95th(us): 6000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f05f6e4a-8758-47eb-896b-09c2c397375e {"id":"f05f6e4a-8758-47eb-896b-09c2c397375e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690116648969,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f05f6e4a-8758-47eb-896b-09c2c397375e {"id":"f05f6e4a-8758-47eb-896b-09c2c397375e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690116648969,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f05f6e4a-8758-47eb-896b-09c2c397375e {"id":"f05f6e4a-8758-47eb-896b-09c2c397375e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690116648969,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":2}' + echo '{"count":2}' {"count":2} ++ echo '{"count":2}' ++ jq -r .count + value=2 + '[' -1 == -1 ']' + '[' 2 '!=' null ']' + exit 0 [Fri Apr 4 16:09:25 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1127911281.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3084 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/612ab083-7ea9-4eca-8097-408679c514a4 {"id":"612ab083-7ea9-4eca-8097-408679c514a4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/__cdc_meta__/owner/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":813933754614727390,"high":2004519558964326820},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367015043551002,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:16.943277028+08:00","start-ts":457114688467238913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114689712422920} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690223603725,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/612ab083-7ea9-4eca-8097-408679c514a4 {"id":"612ab083-7ea9-4eca-8097-408679c514a4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/__cdc_meta__/owner/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":813933754614727390,"high":2004519558964326820},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367015043551002,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:16.943277028+08:00","start-ts":457114688467238913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114689712422920} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690223603725,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/612ab083-7ea9-4eca-8097-408679c514a4 {"id":"612ab083-7ea9-4eca-8097-408679c514a4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/__cdc_meta__/owner/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":813933754614727390,"high":2004519558964326820},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367015043551002,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:16.943277028+08:00","start-ts":457114688467238913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114689712422920} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690223603725,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:09:25 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1055610558.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3084 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4526cf4b-fa58-46e5-822e-762e8c6cf80e {"id":"4526cf4b-fa58-46e5-822e-762e8c6cf80e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4559205366755546684,"high":386323633450708134},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367017944090367,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:18.712410265+08:00","start-ts":457114688933593091,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690178514949} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690532671499,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4526cf4b-fa58-46e5-822e-762e8c6cf80e {"id":"4526cf4b-fa58-46e5-822e-762e8c6cf80e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4559205366755546684,"high":386323633450708134},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367017944090367,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:18.712410265+08:00","start-ts":457114688933593091,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690178514949} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690532671499,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4526cf4b-fa58-46e5-822e-762e8c6cf80e {"id":"4526cf4b-fa58-46e5-822e-762e8c6cf80e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4559205366755546684,"high":386323633450708134},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367017944090367,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:18.712410265+08:00","start-ts":457114688933593091,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690178514949} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690532671499,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:09:25 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1309613098.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/971d3c7d-55c5-4d0c-86db-44d25d519b9e {"id":"971d3c7d-55c5-4d0c-86db-44d25d519b9e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/capture/a4a22b5f-9106-4b96-8fca-f25b40cf0948 {"id":"a4a22b5f-9106-4b96-8fca-f25b40cf0948","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754146} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3fda 971d3c7d-55c5-4d0c-86db-44d25d519b9e /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3fda 971d3c7d-55c5-4d0c-86db-44d25d519b9e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7152490815851699041,"high":13728212366313176759},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366940792512338,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:01.705716672+08:00","start-ts":457114684474523649,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114685719707653} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690818408456,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/971d3c7d-55c5-4d0c-86db-44d25d519b9e {"id":"971d3c7d-55c5-4d0c-86db-44d25d519b9e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/capture/a4a22b5f-9106-4b96-8fca-f25b40cf0948 {"id":"a4a22b5f-9106-4b96-8fca-f25b40cf0948","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754146} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3fda 971d3c7d-55c5-4d0c-86db-44d25d519b9e /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3fda 971d3c7d-55c5-4d0c-86db-44d25d519b9e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7152490815851699041,"high":13728212366313176759},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366940792512338,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:01.705716672+08:00","start-ts":457114684474523649,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114685719707653} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690818408456,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/971d3c7d-55c5-4d0c-86db-44d25d519b9e {"id":"971d3c7d-55c5-4d0c-86db-44d25d519b9e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/capture/a4a22b5f-9106-4b96-8fca-f25b40cf0948 {"id":"a4a22b5f-9106-4b96-8fca-f25b40cf0948","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754146} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd85a3fda 971d3c7d-55c5-4d0c-86db-44d25d519b9e /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3cc7 a4a22b5f-9106-4b96-8fca-f25b40cf0948 /tidb/cdc/default/__cdc_meta__/owner/223195ffd85a3fda 971d3c7d-55c5-4d0c-86db-44d25d519b9e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7152490815851699041,"high":13728212366313176759},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366940792512338,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:01.705716672+08:00","start-ts":457114684474523649,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114685719707653} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690818408456,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:26 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2139680-8548-44b3-bfbb-31d63ec8206e {"id":"b2139680-8548-44b3-bfbb-31d63ec8206e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690660597774,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2139680-8548-44b3-bfbb-31d63ec8206e {"id":"b2139680-8548-44b3-bfbb-31d63ec8206e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690660597774,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2139680-8548-44b3-bfbb-31d63ec8206e {"id":"b2139680-8548-44b3-bfbb-31d63ec8206e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690660597774,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:09:26 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1103611038.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:26 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf {"id":"409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf {"id":"409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf {"id":"409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114690996666369 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.10442.out cli changefeed create --start-ts=457114690996666369 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367051100265341,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:26.597229711+08:00","start_ts":457114690996666369,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114690996666369,"checkpoint_ts":457114690996666369,"checkpoint_time":"2025-04-04 16:09:21.822","gid":{"low":4343697654294407245,"high":14354032950356581550}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:26 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81ab59d9-9a4c-4422-b699-c4da928394f8 {"id":"81ab59d9-9a4c-4422-b699-c4da928394f8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81ab59d9-9a4c-4422-b699-c4da928394f8 {"id":"81ab59d9-9a4c-4422-b699-c4da928394f8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81ab59d9-9a4c-4422-b699-c4da928394f8 {"id":"81ab59d9-9a4c-4422-b699-c4da928394f8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114691000336385 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.10459.out cli changefeed create --start-ts=457114691000336385 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367053141610012,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:26.609050952+08:00","start_ts":457114691000336385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114691000336385,"checkpoint_ts":457114691000336385,"checkpoint_time":"2025-04-04 16:09:21.836","gid":{"low":7370088951958399152,"high":13055422429210467463}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later + set +x Command succeeded on attempt 1 + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eedcd348-cafb-41e1-b578-d3c8274eda87 {"id":"eedcd348-cafb-41e1-b578-d3c8274eda87","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754165} /tidb/cdc/default/__cdc_meta__/capture/f05f6e4a-8758-47eb-896b-09c2c397375e {"id":"f05f6e4a-8758-47eb-896b-09c2c397375e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87162eb eedcd348-cafb-41e1-b578-d3c8274eda87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/__cdc_meta__/owner/223195ffd87162eb eedcd348-cafb-41e1-b578-d3c8274eda87 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690116648969,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eedcd348-cafb-41e1-b578-d3c8274eda87 {"id":"eedcd348-cafb-41e1-b578-d3c8274eda87","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754165} /tidb/cdc/default/__cdc_meta__/capture/f05f6e4a-8758-47eb-896b-09c2c397375e {"id":"f05f6e4a-8758-47eb-896b-09c2c397375e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87162eb eedcd348-cafb-41e1-b578-d3c8274eda87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/__cdc_meta__/owner/223195ffd87162eb eedcd348-cafb-41e1-b578-d3c8274eda87 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690116648969,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eedcd348-cafb-41e1-b578-d3c8274eda87 {"id":"eedcd348-cafb-41e1-b578-d3c8274eda87","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754165} /tidb/cdc/default/__cdc_meta__/capture/f05f6e4a-8758-47eb-896b-09c2c397375e {"id":"f05f6e4a-8758-47eb-896b-09c2c397375e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716150 46aaf55a-d22e-4a4b-81cc-137cc90088f1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87162eb eedcd348-cafb-41e1-b578-d3c8274eda87 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8716283 f05f6e4a-8758-47eb-896b-09c2c397375e /tidb/cdc/default/__cdc_meta__/owner/223195ffd87162eb eedcd348-cafb-41e1-b578-d3c8274eda87 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10468490312239419612,"high":6752150446206420402},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366969832589403,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:07.085549337+08:00","start-ts":457114685883285505,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687128469512} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690116648969,"progress":1}' + grep -q 'etcd info' + break + set +x check diff successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10331) - No such process wait process 10331 exit for 1-th time... process 10331 already exit command: check_no_capture http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3665 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/612ab083-7ea9-4eca-8097-408679c514a4 {"id":"612ab083-7ea9-4eca-8097-408679c514a4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/capture/bd1c5e35-9e7e-4225-a6c6-566d73f72efd {"id":"bd1c5e35-9e7e-4225-a6c6-566d73f72efd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754165} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/__cdc_meta__/owner/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/__cdc_meta__/owner/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":813933754614727390,"high":2004519558964326820},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367015043551002,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:16.943277028+08:00","start-ts":457114688467238913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114689712422920} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690223603725,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/612ab083-7ea9-4eca-8097-408679c514a4 {"id":"612ab083-7ea9-4eca-8097-408679c514a4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/capture/bd1c5e35-9e7e-4225-a6c6-566d73f72efd {"id":"bd1c5e35-9e7e-4225-a6c6-566d73f72efd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754165} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/__cdc_meta__/owner/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/__cdc_meta__/owner/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":813933754614727390,"high":2004519558964326820},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367015043551002,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:16.943277028+08:00","start-ts":457114688467238913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114689712422920} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690223603725,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/612ab083-7ea9-4eca-8097-408679c514a4 {"id":"612ab083-7ea9-4eca-8097-408679c514a4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754162} /tidb/cdc/default/__cdc_meta__/capture/bd1c5e35-9e7e-4225-a6c6-566d73f72efd {"id":"bd1c5e35-9e7e-4225-a6c6-566d73f72efd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754165} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895eee3 2dd45c5b-31a4-4e63-83bd-fb2df70b11a4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/__cdc_meta__/owner/223195ffd895efeb 612ab083-7ea9-4eca-8097-408679c514a4 /tidb/cdc/default/__cdc_meta__/owner/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":813933754614727390,"high":2004519558964326820},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367015043551002,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:16.943277028+08:00","start-ts":457114688467238913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114689712422920} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690223603725,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.10625.out cli capture list + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3665 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4526cf4b-fa58-46e5-822e-762e8c6cf80e {"id":"4526cf4b-fa58-46e5-822e-762e8c6cf80e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/capture/65178160-0ea9-4719-9b22-9b2cafb35184 {"id":"65178160-0ea9-4719-9b22-9b2cafb35184","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754166} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea22a 65178160-0ea9-4719-9b22-9b2cafb35184 /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea22a 65178160-0ea9-4719-9b22-9b2cafb35184 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4559205366755546684,"high":386323633450708134},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367017944090367,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:18.712410265+08:00","start-ts":457114688933593091,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690178514949} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692367417353,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4526cf4b-fa58-46e5-822e-762e8c6cf80e {"id":"4526cf4b-fa58-46e5-822e-762e8c6cf80e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/capture/65178160-0ea9-4719-9b22-9b2cafb35184 {"id":"65178160-0ea9-4719-9b22-9b2cafb35184","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754166} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea22a 65178160-0ea9-4719-9b22-9b2cafb35184 /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea22a 65178160-0ea9-4719-9b22-9b2cafb35184 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4559205366755546684,"high":386323633450708134},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367017944090367,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:18.712410265+08:00","start-ts":457114688933593091,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690178514949} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692367417353,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4526cf4b-fa58-46e5-822e-762e8c6cf80e {"id":"4526cf4b-fa58-46e5-822e-762e8c6cf80e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/capture/65178160-0ea9-4719-9b22-9b2cafb35184 {"id":"65178160-0ea9-4719-9b22-9b2cafb35184","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754166} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea0e8 68b4f6cb-9d5a-49f2-b8ef-7e2f788e2b1a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea22a 65178160-0ea9-4719-9b22-9b2cafb35184 /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea22a 65178160-0ea9-4719-9b22-9b2cafb35184 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4559205366755546684,"high":386323633450708134},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367017944090367,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:18.712410265+08:00","start-ts":457114688933593091,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690178514949} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692367417353,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.13173.out cli capture list parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 4 16:09:29 CST 2025] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.1058710589.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:29 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2139680-8548-44b3-bfbb-31d63ec8206e {"id":"b2139680-8548-44b3-bfbb-31d63ec8206e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/capture/c3fc6c67-9f0f-4003-910c-a0c63ed42c81 {"id":"c3fc6c67-9f0f-4003-910c-a0c63ed42c81","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754166} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05b25 c3fc6c67-9f0f-4003-910c-a0c63ed42c81 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05b25 c3fc6c67-9f0f-4003-910c-a0c63ed42c81 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690660597774,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2139680-8548-44b3-bfbb-31d63ec8206e {"id":"b2139680-8548-44b3-bfbb-31d63ec8206e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/capture/c3fc6c67-9f0f-4003-910c-a0c63ed42c81 {"id":"c3fc6c67-9f0f-4003-910c-a0c63ed42c81","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754166} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05b25 c3fc6c67-9f0f-4003-910c-a0c63ed42c81 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05b25 c3fc6c67-9f0f-4003-910c-a0c63ed42c81 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690660597774,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2139680-8548-44b3-bfbb-31d63ec8206e {"id":"b2139680-8548-44b3-bfbb-31d63ec8206e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/capture/c3fc6c67-9f0f-4003-910c-a0c63ed42c81 {"id":"c3fc6c67-9f0f-4003-910c-a0c63ed42c81","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754166} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a059e2 dbb4eddc-2e15-4280-83de-de1d42d8ded1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05b25 c3fc6c67-9f0f-4003-910c-a0c63ed42c81 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05b25 c3fc6c67-9f0f-4003-910c-a0c63ed42c81 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114690660597774,"progress":1}' + grep -q 'etcd info' + break + set +x check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367092301519374 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6420f80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:23337, start at 2025-04-04 16:09:28.391826829 +0800 CST m=+3.277874147 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:28.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:28.382 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:28.382 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367092301519374 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6420f80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:23337, start at 2025-04-04 16:09:28.391826829 +0800 CST m=+3.277874147 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:28.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:28.382 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:28.382 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367094098929266 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6423f00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:23412, start at 2025-04-04 16:09:28.583336938 +0800 CST m=+3.295967884 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:28.590 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:28.572 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:28.572 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table fail_over_ddl_test.test2 not exists for 1-th check, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 table fail_over_ddl_test.test2 exists check diff failed 1-th time, retry later + set +x + ans='[ { "id": "612ab083-7ea9-4eca-8097-408679c514a4", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "bd1c5e35-9e7e-4225-a6c6-566d73f72efd", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"612ab083-7ea9-4eca-8097-408679c514a4",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"bd1c5e35-9e7e-4225-a6c6-566d73f72efd",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=bd1c5e35-9e7e-4225-a6c6-566d73f72efd + '[' -z bd1c5e35-9e7e-4225-a6c6-566d73f72efd ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d bd1c5e35-9e7e-4225-a6c6-566d73f72efd + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.10654.out cli changefeed move-table -c test -t 114 -d bd1c5e35-9e7e-4225-a6c6-566d73f72efd ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":2}' + echo '{"count":2}' {"count":2} ++ echo '{"count":2}' ++ jq -r .count + value=2 + '[' -1 == -1 ']' + '[' 2 '!=' null ']' + exit 0 [Fri Apr 4 16:09:28 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.88288830.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseJ-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + ans='[ { "id": "4526cf4b-fa58-46e5-822e-762e8c6cf80e", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "65178160-0ea9-4719-9b22-9b2cafb35184", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"4526cf4b-fa58-46e5-822e-762e8c6cf80e",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"65178160-0ea9-4719-9b22-9b2cafb35184",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=65178160-0ea9-4719-9b22-9b2cafb35184 + '[' -z 65178160-0ea9-4719-9b22-9b2cafb35184 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 65178160-0ea9-4719-9b22-9b2cafb35184 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.13205.out cli changefeed move-table -c test -t 114 -d 65178160-0ea9-4719-9b22-9b2cafb35184 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10399) - No such process wait process 10399 exit for 1-th time... process 10399 already exit [Fri Apr 4 16:09:30 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1053510537.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367100736289111 TiDB Cluster ID. tikv_gc_leader_uuid 657ff643fbc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:9539, start at 2025-04-04 16:09:30.361947908 +0800 CST m=+3.489445730 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:30.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:30.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:30.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367100736289111 TiDB Cluster ID. tikv_gc_leader_uuid 657ff643fbc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:9539, start at 2025-04-04 16:09:30.361947908 +0800 CST m=+3.489445730 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:30.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:30.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:30.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367104343790131 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6442c80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:9604, start at 2025-04-04 16:09:30.556635227 +0800 CST m=+3.527285332 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:30.564 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:30.596 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:30.596 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table fail_over_ddl_test.test2 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03ac071f-0323-449f-9dcd-f3621d78b349 {"id":"03ac071f-0323-449f-9dcd-f3621d78b349","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754128} /tidb/cdc/default/__cdc_meta__/capture/42ef0bc6-17bc-4a84-a999-f12e0d3085bd {"id":"42ef0bc6-17bc-4a84-a999-f12e0d3085bd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754168} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd811194e 42ef0bc6-17bc-4a84-a999-f12e0d3085bd /tidb/cdc/default/__cdc_meta__/owner/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/__cdc_meta__/owner/223195ffd811194e 42ef0bc6-17bc-4a84-a999-f12e0d3085bd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12773142019420507618,"high":17286550619415136416},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366861311330212,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:41.37280854+08:00","start-ts":457114679144873985,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114680390057990} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114693379293196,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03ac071f-0323-449f-9dcd-f3621d78b349 {"id":"03ac071f-0323-449f-9dcd-f3621d78b349","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754128} /tidb/cdc/default/__cdc_meta__/capture/42ef0bc6-17bc-4a84-a999-f12e0d3085bd {"id":"42ef0bc6-17bc-4a84-a999-f12e0d3085bd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754168} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd811194e 42ef0bc6-17bc-4a84-a999-f12e0d3085bd /tidb/cdc/default/__cdc_meta__/owner/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/__cdc_meta__/owner/223195ffd811194e 42ef0bc6-17bc-4a84-a999-f12e0d3085bd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12773142019420507618,"high":17286550619415136416},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366861311330212,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:41.37280854+08:00","start-ts":457114679144873985,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114680390057990} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114693379293196,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03ac071f-0323-449f-9dcd-f3621d78b349 {"id":"03ac071f-0323-449f-9dcd-f3621d78b349","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754128} /tidb/cdc/default/__cdc_meta__/capture/42ef0bc6-17bc-4a84-a999-f12e0d3085bd {"id":"42ef0bc6-17bc-4a84-a999-f12e0d3085bd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754168} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd811194e 42ef0bc6-17bc-4a84-a999-f12e0d3085bd /tidb/cdc/default/__cdc_meta__/owner/223195ffd8111427 03ac071f-0323-449f-9dcd-f3621d78b349 /tidb/cdc/default/__cdc_meta__/owner/223195ffd811194e 42ef0bc6-17bc-4a84-a999-f12e0d3085bd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12773142019420507618,"high":17286550619415136416},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366861311330212,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:08:41.37280854+08:00","start-ts":457114679144873985,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114680390057990} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114693379293196,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10393) - No such process wait process 10393 exit for 1-th time... process 10393 already exit [Fri Apr 4 16:09:32 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1051310515.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.24635.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3209 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58c11669-e9f6-4d79-a055-6db77147fa95 {"id":"58c11669-e9f6-4d79-a055-6db77147fa95","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754169} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bf6cda 30b07806-981c-451c-9850-41d31a488ccb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bf6df2 58c11669-e9f6-4d79-a055-6db77147fa95 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bf6df2 58c11669-e9f6-4d79-a055-6db77147fa95 /tidb/cdc/default/default/changefeed/info/1400012301752781067317958359255394092425 {"id":{"id":{"low":14000123017527810673,"high":17958359255394092425},"display":{"name":"1400012301752781067317958359255394092425","namespace":"default"}},"upstream-id":7489367050049368721,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:09:23.179898599+08:00","start-ts":457114691344007178,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114691344007183} /tidb/cdc/default/default/changefeed/status/1400012301752781067317958359255394092425 {"checkpoint-ts":457114692484333576,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58c11669-e9f6-4d79-a055-6db77147fa95 {"id":"58c11669-e9f6-4d79-a055-6db77147fa95","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754169} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bf6cda 30b07806-981c-451c-9850-41d31a488ccb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bf6df2 58c11669-e9f6-4d79-a055-6db77147fa95 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bf6df2 58c11669-e9f6-4d79-a055-6db77147fa95 /tidb/cdc/default/default/changefeed/info/1400012301752781067317958359255394092425 {"id":{"id":{"low":14000123017527810673,"high":17958359255394092425},"display":{"name":"1400012301752781067317958359255394092425","namespace":"default"}},"upstream-id":7489367050049368721,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:09:23.179898599+08:00","start-ts":457114691344007178,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114691344007183} /tidb/cdc/default/default/changefeed/status/1400012301752781067317958359255394092425 {"checkpoint-ts":457114692484333576,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58c11669-e9f6-4d79-a055-6db77147fa95 {"id":"58c11669-e9f6-4d79-a055-6db77147fa95","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754169} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bf6cda 30b07806-981c-451c-9850-41d31a488ccb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bf6df2 58c11669-e9f6-4d79-a055-6db77147fa95 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bf6df2 58c11669-e9f6-4d79-a055-6db77147fa95 /tidb/cdc/default/default/changefeed/info/1400012301752781067317958359255394092425 {"id":{"id":{"low":14000123017527810673,"high":17958359255394092425},"display":{"name":"1400012301752781067317958359255394092425","namespace":"default"}},"upstream-id":7489367050049368721,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T16:09:23.179898599+08:00","start-ts":457114691344007178,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114691344007183} /tidb/cdc/default/default/changefeed/status/1400012301752781067317958359255394092425 {"checkpoint-ts":457114692484333576,"progress":1}' + grep -q 'etcd info' + break + set +x command: cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=4 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 check diff failed 2-th time, retry later + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/^=== Command to ticdc(new arch). //' ++ sed 's/ PASS.*//' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCase-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Run 10 test check diff failed 1-th time, retry later check diff failed 2-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.cli.10930.out cli tso query --pd=http://127.0.0.1:2379 command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x + tso='457114693697273857 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114693697273857 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:33 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.2467224674.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 "id": "58c11669-e9f6-4d79-a055-6db77147fa95", "cluster_id": "default" run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:33 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 {"id":"adbdf3ff-3990-4a28-8d70-e0f62c4d89f9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754171} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7370088951958399152,"high":13055422429210467463},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367053141610012,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.609050952+08:00","start-ts":457114691000336385,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692245520391} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692625629214,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 {"id":"adbdf3ff-3990-4a28-8d70-e0f62c4d89f9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754171} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7370088951958399152,"high":13055422429210467463},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367053141610012,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.609050952+08:00","start-ts":457114691000336385,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692245520391} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692625629214,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 {"id":"adbdf3ff-3990-4a28-8d70-e0f62c4d89f9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754171} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7370088951958399152,"high":13055422429210467463},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367053141610012,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.609050952+08:00","start-ts":457114691000336385,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692245520391} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692625629214,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:09:33 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1060710609.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 16:09:33.859 +08:00] [DEBUG] [pd.rs:582] ["update latency statistics for raftstore-v1"] [tick=425] [thread_id=88] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseG-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory capture_id: 58c11669-e9f6-4d79-a055-6db77147fa95 + set +x + tso='457114694136889345 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114694136889345 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:35 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1096410966.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46c9cdef-cbdc-4689-817c-f5e1ea80f444 {"id":"46c9cdef-cbdc-4689-817c-f5e1ea80f444","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754172} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4343697654294407245,"high":14354032950356581550},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367051100265341,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.597229711+08:00","start-ts":457114690996666369,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692242112520} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692753293335,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46c9cdef-cbdc-4689-817c-f5e1ea80f444 {"id":"46c9cdef-cbdc-4689-817c-f5e1ea80f444","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754172} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4343697654294407245,"high":14354032950356581550},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367051100265341,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.597229711+08:00","start-ts":457114690996666369,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692242112520} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692753293335,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46c9cdef-cbdc-4689-817c-f5e1ea80f444 {"id":"46c9cdef-cbdc-4689-817c-f5e1ea80f444","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754172} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4343697654294407245,"high":14354032950356581550},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367051100265341,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.597229711+08:00","start-ts":457114690996666369,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692242112520} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692753293335,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:09:35 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1059110593.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_N/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 3-th time, retry later command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 [2025/04/04 16:09:36.184 +08:00] [DEBUG] [peer.rs:2639] ["renew lease"] [reason=tick] [peer_id=148] [region_id=147] [thread_id=82] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCase-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_J Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:36 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e632ff5-353b-4505-bb1f-39bb194bdabb {"id":"0e632ff5-353b-4505-bb1f-39bb194bdabb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754173} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e632ff5-353b-4505-bb1f-39bb194bdabb {"id":"0e632ff5-353b-4505-bb1f-39bb194bdabb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754173} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e632ff5-353b-4505-bb1f-39bb194bdabb {"id":"0e632ff5-353b-4505-bb1f-39bb194bdabb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754173} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114693697273857 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.24747.out cli changefeed create --start-ts=457114693697273857 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367092301519374,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:36.905507251+08:00","start_ts":457114693697273857,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114693697273857,"checkpoint_ts":457114693697273857,"checkpoint_time":"2025-04-04 16:09:32.124","gid":{"low":8380935837768221898,"high":1525368754776906891}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... ***************** properties ***************** "updateproportion"="0" "mysql.db"="changefeed_reconstruct_new" "requestdistribution"="uniform" "recordcount"="50" "scanproportion"="0" "insertproportion"="0" "mysql.user"="root" "workload"="core" "operationcount"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "threadcount"="4" "dotransactions"="false" "readproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 18.376609ms INSERT - Takes(s): 0.0, Count: 46, OPS: 3548.0, Avg(us): 1284, Min(us): 878, Max(us): 5317, 95th(us): 2000, 99th(us): 6000 table changefeed_reconstruct_new.usertable exists check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3eeb65a4-eefe-4400-8b25-56e25cd9381d {"id":"3eeb65a4-eefe-4400-8b25-56e25cd9381d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754174} /tidb/cdc/default/__cdc_meta__/capture/adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 {"id":"adbdf3ff-3990-4a28-8d70-e0f62c4d89f9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754171} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c18a28 3eeb65a4-eefe-4400-8b25-56e25cd9381d /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c18a28 3eeb65a4-eefe-4400-8b25-56e25cd9381d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7370088951958399152,"high":13055422429210467463},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367053141610012,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.609050952+08:00","start-ts":457114691000336385,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692245520391} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692625629214,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3eeb65a4-eefe-4400-8b25-56e25cd9381d {"id":"3eeb65a4-eefe-4400-8b25-56e25cd9381d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754174} /tidb/cdc/default/__cdc_meta__/capture/adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 {"id":"adbdf3ff-3990-4a28-8d70-e0f62c4d89f9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754171} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c18a28 3eeb65a4-eefe-4400-8b25-56e25cd9381d /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c18a28 3eeb65a4-eefe-4400-8b25-56e25cd9381d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7370088951958399152,"high":13055422429210467463},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367053141610012,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.609050952+08:00","start-ts":457114691000336385,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692245520391} /tidb/cdc/default/default/changefeed/status/test + grep -q 'failed to get info:' {"checkpoint-ts":457114692625629214,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3eeb65a4-eefe-4400-8b25-56e25cd9381d {"id":"3eeb65a4-eefe-4400-8b25-56e25cd9381d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754174} /tidb/cdc/default/__cdc_meta__/capture/adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 {"id":"adbdf3ff-3990-4a28-8d70-e0f62c4d89f9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754171} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c188e6 81ab59d9-9a4c-4422-b699-c4da928394f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c18a28 3eeb65a4-eefe-4400-8b25-56e25cd9381d /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c18a28 3eeb65a4-eefe-4400-8b25-56e25cd9381d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7370088951958399152,"high":13055422429210467463},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367053141610012,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.609050952+08:00","start-ts":457114691000336385,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692245520391} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692625629214,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.10678.out cli capture list [2025/04/04 16:09:33.859 +08:00] [DEBUG] chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Command succeeded on attempt 1 + set +x + ans='[ { "id": "adbdf3ff-3990-4a28-8d70-e0f62c4d89f9", table fail_over_ddl_test.test2 not exists for 1-th check, retry later "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "3eeb65a4-eefe-4400-8b25-56e25cd9381d", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"adbdf3ff-3990-4a28-8d70-e0f62c4d89f9",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"3eeb65a4-eefe-4400-8b25-56e25cd9381d",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=3eeb65a4-eefe-4400-8b25-56e25cd9381d + '[' -z 3eeb65a4-eefe-4400-8b25-56e25cd9381d ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 3eeb65a4-eefe-4400-8b25-56e25cd9381d + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.10707.out cli changefeed move-table -c test -t 114 -d 3eeb65a4-eefe-4400-8b25-56e25cd9381d + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:38 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e5219c5-be34-4228-9a65-1784be762b9e {"id":"3e5219c5-be34-4228-9a65-1784be762b9e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754175} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e5219c5-be34-4228-9a65-1784be762b9e {"id":"3e5219c5-be34-4228-9a65-1784be762b9e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754175} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e5219c5-be34-4228-9a65-1784be762b9e {"id":"3e5219c5-be34-4228-9a65-1784be762b9e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754175} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114694136889345 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.cli.11030.out cli changefeed create --start-ts=457114694136889345 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367100736289111,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:38.585896555+08:00","start_ts":457114694136889345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114694136889345,"checkpoint_ts":457114694136889345,"checkpoint_time":"2025-04-04 16:09:33.801","gid":{"low":11043631044976089295,"high":5444269735529934263}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:38 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46c9cdef-cbdc-4689-817c-f5e1ea80f444 {"id":"46c9cdef-cbdc-4689-817c-f5e1ea80f444","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754172} /tidb/cdc/default/__cdc_meta__/capture/56c08182-1570-4d6f-ac69-a63e4448e3b9 {"id":"56c08182-1570-4d6f-ac69-a63e4448e3b9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754175} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd6346 56c08182-1570-4d6f-ac69-a63e4448e3b9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd6346 56c08182-1570-4d6f-ac69-a63e4448e3b9 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4343697654294407245,"high":14354032950356581550},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367051100265341,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.597229711+08:00","start-ts":457114690996666369,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692242112520} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692753293335,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46c9cdef-cbdc-4689-817c-f5e1ea80f444 {"id":"46c9cdef-cbdc-4689-817c-f5e1ea80f444","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754172} /tidb/cdc/default/__cdc_meta__/capture/56c08182-1570-4d6f-ac69-a63e4448e3b9 {"id":"56c08182-1570-4d6f-ac69-a63e4448e3b9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754175} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd6346 56c08182-1570-4d6f-ac69-a63e4448e3b9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd6346 56c08182-1570-4d6f-ac69-a63e4448e3b9 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4343697654294407245,"high":14354032950356581550},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367051100265341,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.597229711+08:00","start-ts":457114690996666369,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692242112520} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692753293335,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46c9cdef-cbdc-4689-817c-f5e1ea80f444 {"id":"46c9cdef-cbdc-4689-817c-f5e1ea80f444","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754172} /tidb/cdc/default/__cdc_meta__/capture/56c08182-1570-4d6f-ac69-a63e4448e3b9 {"id":"56c08182-1570-4d6f-ac69-a63e4448e3b9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754175} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd61da 409a9bfb-e8ae-4c2d-bc81-db60c9b5eaaf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd6346 56c08182-1570-4d6f-ac69-a63e4448e3b9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd6346 56c08182-1570-4d6f-ac69-a63e4448e3b9 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4343697654294407245,"high":14354032950356581550},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367051100265341,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.597229711+08:00","start-ts":457114690996666369,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692242112520} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114692753293335,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.10662.out cli capture list chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_N Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff successfully Verifying downstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/changefeed_reconstruct/stdoutserver1.log /tmp/tidb_cdc_test/changefeed_reconstruct/stdoutserver2.log no DATA RACE found [Fri Apr 4 16:09:39 CST 2025] <<<<<< run test case changefeed_reconstruct success! >>>>>> + set +x Command succeeded on attempt 1 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + ans='[ { "id": "46c9cdef-cbdc-4689-817c-f5e1ea80f444", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "56c08182-1570-4d6f-ac69-a63e4448e3b9", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"46c9cdef-cbdc-4689-817c-f5e1ea80f444",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"56c08182-1570-4d6f-ac69-a63e4448e3b9",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=56c08182-1570-4d6f-ac69-a63e4448e3b9 + '[' -z 56c08182-1570-4d6f-ac69-a63e4448e3b9 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 56c08182-1570-4d6f-ac69-a63e4448e3b9 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.10694.out cli changefeed move-table -c test -t 114 -d 56c08182-1570-4d6f-ac69-a63e4448e3b9 table fail_over_ddl_test.test2 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24677) - No such process wait process 24677 exit for 1-th time... process 24677 already exit [Fri Apr 4 16:09:40 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.2481524817.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_C Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table fail_over_ddl_test.test2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_G Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully table fail_over_ddl_test.test2 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:44 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dad8d493-223f-46a3-b155-11fcbcf08790 {"id":"dad8d493-223f-46a3-b155-11fcbcf08790","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754181} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695335411733,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dad8d493-223f-46a3-b155-11fcbcf08790 {"id":"dad8d493-223f-46a3-b155-11fcbcf08790","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754181} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695335411733,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dad8d493-223f-46a3-b155-11fcbcf08790 {"id":"dad8d493-223f-46a3-b155-11fcbcf08790","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754181} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695335411733,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:09:44 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.2488824890.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10969) - No such process wait process 10969 exit for 1-th time... process 10969 already exit [Fri Apr 4 16:09:44 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1110111103.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_H Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367166296562395 TiDB Cluster ID. tikv_gc_leader_uuid 657ff652b200005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:11476, start at 2025-04-04 16:09:45.421199375 +0800 CST m=+3.553018835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:45.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:45.416 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:45.416 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367166296562395 TiDB Cluster ID. tikv_gc_leader_uuid 657ff652b200005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:11476, start at 2025-04-04 16:09:45.421199375 +0800 CST m=+3.553018835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:45.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:45.416 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:45.416 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367170741610204 TiDB Cluster ID. tikv_gc_leader_uuid 657ff652d400006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:11557, start at 2025-04-04 16:09:45.560802386 +0800 CST m=+3.529810895 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:45.569 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:45.552 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:45.552 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13262) - No such process wait process 13262 exit for 1-th time... process 13262 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dad8d493-223f-46a3-b155-11fcbcf08790 {"id":"dad8d493-223f-46a3-b155-11fcbcf08790","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754181} /tidb/cdc/default/__cdc_meta__/capture/e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 {"id":"e6ae11c6-ee6c-49b4-94bd-5df936cf8b38","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754184} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53b29 e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53b29 e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695335411733,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dad8d493-223f-46a3-b155-11fcbcf08790 {"id":"dad8d493-223f-46a3-b155-11fcbcf08790","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754181} /tidb/cdc/default/__cdc_meta__/capture/e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 {"id":"e6ae11c6-ee6c-49b4-94bd-5df936cf8b38","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754184} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53b29 e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53b29 e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695335411733,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dad8d493-223f-46a3-b155-11fcbcf08790 {"id":"dad8d493-223f-46a3-b155-11fcbcf08790","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754181} /tidb/cdc/default/__cdc_meta__/capture/e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 {"id":"e6ae11c6-ee6c-49b4-94bd-5df936cf8b38","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754184} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e539ea 0e632ff5-353b-4505-bb1f-39bb194bdabb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53b29 e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53b29 e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695335411733,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.24961.out cli capture list + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca3d1581-3f09-42c5-ac59-67d87237b63c {"id":"ca3d1581-3f09-42c5-ac59-67d87237b63c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754184} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695906361347,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca3d1581-3f09-42c5-ac59-67d87237b63c {"id":"ca3d1581-3f09-42c5-ac59-67d87237b63c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754184} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695906361347,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca3d1581-3f09-42c5-ac59-67d87237b63c {"id":"ca3d1581-3f09-42c5-ac59-67d87237b63c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754184} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695906361347,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:09:47 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1115911161.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10487) - No such process wait process 10487 exit for 1-th time... process 10487 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.12841.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + ans='[ { "id": "dad8d493-223f-46a3-b155-11fcbcf08790", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "e6ae11c6-ee6c-49b4-94bd-5df936cf8b38", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"dad8d493-223f-46a3-b155-11fcbcf08790",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"e6ae11c6-ee6c-49b4-94bd-5df936cf8b38",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 + '[' -z e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.24998.out cli changefeed move-table -c test -t 114 -d e6ae11c6-ee6c-49b4-94bd-5df936cf8b38 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367178140310957 TiDB Cluster ID. tikv_gc_leader_uuid 657ff654c0c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:8175, start at 2025-04-04 16:09:47.532630725 +0800 CST m=+3.138790848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:47.539 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:47.545 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:47.545 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367178140310957 TiDB Cluster ID. tikv_gc_leader_uuid 657ff654c0c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:8175, start at 2025-04-04 16:09:47.532630725 +0800 CST m=+3.138790848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:47.539 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:47.545 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:47.545 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367181472432654 TiDB Cluster ID. tikv_gc_leader_uuid 657ff654eb80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:8249, start at 2025-04-04 16:09:47.733680873 +0800 CST m=+3.192843128 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:47.740 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:47.743 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:47.743 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 91000 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114698071408642 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114698071408642 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:50 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1287012872.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367088931747056 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6553c8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:9552, start at 2025-04-04 16:09:48.054147176 +0800 CST m=+24.055604107 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:48.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:48.018 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:48.018 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367088931747056 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6553c8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:9552, start at 2025-04-04 16:09:48.054147176 +0800 CST m=+24.055604107 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:48.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:48.018 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:48.018 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367092583820341 TiDB Cluster ID. tikv_gc_leader_uuid 657ff640178000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-sddr0--dd5d9, pid:9607, start at 2025-04-04 16:09:26.414457786 +0800 CST m=+2.314332530 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:26.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:26.466 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:26.466 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/639cb136-0305-4556-aa0a-957f609567ae {"id":"639cb136-0305-4556-aa0a-957f609567ae","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754187} /tidb/cdc/default/__cdc_meta__/capture/ca3d1581-3f09-42c5-ac59-67d87237b63c {"id":"ca3d1581-3f09-42c5-ac59-67d87237b63c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754184} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2d54 639cb136-0305-4556-aa0a-957f609567ae /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2d54 639cb136-0305-4556-aa0a-957f609567ae /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695906361347,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/639cb136-0305-4556-aa0a-957f609567ae {"id":"639cb136-0305-4556-aa0a-957f609567ae","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754187} /tidb/cdc/default/__cdc_meta__/capture/ca3d1581-3f09-42c5-ac59-67d87237b63c {"id":"ca3d1581-3f09-42c5-ac59-67d87237b63c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754184} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2d54 639cb136-0305-4556-aa0a-957f609567ae /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2d54 639cb136-0305-4556-aa0a-957f609567ae /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695906361347,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/639cb136-0305-4556-aa0a-957f609567ae {"id":"639cb136-0305-4556-aa0a-957f609567ae","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754187} /tidb/cdc/default/__cdc_meta__/capture/ca3d1581-3f09-42c5-ac59-67d87237b63c {"id":"ca3d1581-3f09-42c5-ac59-67d87237b63c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754184} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2be9 3e5219c5-be34-4228-9a65-1784be762b9e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2d54 639cb136-0305-4556-aa0a-957f609567ae /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec2d54 639cb136-0305-4556-aa0a-957f609567ae /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695906361347,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367190626617227 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6570300003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:14263, start at 2025-04-04 16:09:49.837956392 +0800 CST m=+3.243223537 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:49.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:49.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:49.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367190626617227 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6570300003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:14263, start at 2025-04-04 16:09:49.837956392 +0800 CST m=+3.243223537 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:49.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:49.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:49.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367183665319895 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6577540004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:14331, start at 2025-04-04 16:09:50.340579154 +0800 CST m=+3.582379570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:50.348 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:50.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:50.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.9538.out cli tso query --pd=http://127.0.0.1:2379 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10968) - No such process wait process 10968 exit for 1-th time... process 10968 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (11041) - No such process wait process 11041 exit for 1-th time... process 11041 already exit [Fri Apr 4 16:09:50 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1114111143.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":2}' + echo '{"count":2}' {"count":2} ++ echo '{"count":2}' ++ jq -r .count + value=2 + '[' -1 == -1 ']' + '[' 2 '!=' null ']' + exit 0 [Fri Apr 4 16:09:51 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1353413536.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_A/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13101) - No such process wait process 13101 exit for 1-th time... process 13101 already exit [Fri Apr 4 16:09:48 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1327513277.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:51 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3562 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/108b1db9-4f87-45d7-9903-da4e7337c3f7 {"id":"108b1db9-4f87-45d7-9903-da4e7337c3f7","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754189} /tidb/cdc/default/__cdc_meta__/capture/4526cf4b-fa58-46e5-822e-762e8c6cf80e {"id":"4526cf4b-fa58-46e5-822e-762e8c6cf80e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea686 108b1db9-4f87-45d7-9903-da4e7337c3f7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea686 108b1db9-4f87-45d7-9903-da4e7337c3f7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4559205366755546684,"high":386323633450708134},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367017944090367,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:18.712410265+08:00","start-ts":457114688933593091,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690178514949} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114693940543498,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/108b1db9-4f87-45d7-9903-da4e7337c3f7 {"id":"108b1db9-4f87-45d7-9903-da4e7337c3f7","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754189} /tidb/cdc/default/__cdc_meta__/capture/4526cf4b-fa58-46e5-822e-762e8c6cf80e {"id":"4526cf4b-fa58-46e5-822e-762e8c6cf80e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea686 108b1db9-4f87-45d7-9903-da4e7337c3f7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea686 108b1db9-4f87-45d7-9903-da4e7337c3f7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4559205366755546684,"high":386323633450708134},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367017944090367,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:18.712410265+08:00","start-ts":457114688933593091,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690178514949} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114693940543498,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/108b1db9-4f87-45d7-9903-da4e7337c3f7 {"id":"108b1db9-4f87-45d7-9903-da4e7337c3f7","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754189} /tidb/cdc/default/__cdc_meta__/capture/4526cf4b-fa58-46e5-822e-762e8c6cf80e {"id":"4526cf4b-fa58-46e5-822e-762e8c6cf80e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754163} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd89ea686 108b1db9-4f87-45d7-9903-da4e7337c3f7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea1c6 4526cf4b-fa58-46e5-822e-762e8c6cf80e /tidb/cdc/default/__cdc_meta__/owner/223195ffd89ea686 108b1db9-4f87-45d7-9903-da4e7337c3f7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4559205366755546684,"high":386323633450708134},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367017944090367,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:18.712410265+08:00","start-ts":457114688933593091,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690178514949} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114693940543498,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":1}' + echo '{"count":1}' {"count":1} ++ echo '{"count":1}' ++ jq -r .count + value=1 + '[' -1 == -1 ']' + '[' 1 '!=' null ']' + exit 0 [Fri Apr 4 16:09:52 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1076510767.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 4 16:09:52 CST 2025] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/coordinator/InjectUpdateGCTickerInterval=return(100)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.1086510867.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114698736205825 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114698736205825 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:52 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.95809582.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367200255130049 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6595840009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:12272, start at 2025-04-04 16:09:52.233188132 +0800 CST m=+3.588759151 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:52.240 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:52.225 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:52.225 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367200255130049 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6595840009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:12272, start at 2025-04-04 16:09:52.233188132 +0800 CST m=+3.588759151 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:52.240 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:52.225 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:52.225 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367200149189456 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6594180006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:12340, start at 2025-04-04 16:09:52.141717888 +0800 CST m=+3.389993908 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:52.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:52.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:52.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:53 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 {"id":"7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754190} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114697751592962,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 {"id":"7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754190} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test + grep -q 'failed to get info:' {"checkpoint-ts":457114697751592962,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 {"id":"7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754190} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114697751592962,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":2}' + echo '{"count":2}' {"count":2} ++ echo '{"count":2}' ++ jq -r .count + value=2 + '[' -1 == -1 ']' + '[' 2 '!=' null ']' + exit 0 [Fri Apr 4 16:09:53 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1121211214.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:53 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 {"id":"254bedfe-dcd6-42e7-a4b5-ccd9f7db7690","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754190} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 {"id":"254bedfe-dcd6-42e7-a4b5-ccd9f7db7690","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754190} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 {"id":"254bedfe-dcd6-42e7-a4b5-ccd9f7db7690","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754190} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114698071408642 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.12936.out cli changefeed create --start-ts=457114698071408642 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367166296562395,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:53.5448803+08:00","start_ts":457114698071408642,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114698071408642,"checkpoint_ts":457114698071408642,"checkpoint_time":"2025-04-04 16:09:48.810","gid":{"low":5065050923354093612,"high":7008916332082200506}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.15694.out cli tso query --pd=http://127.0.0.1:2379 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367202340662815 TiDB Cluster ID. tikv_gc_leader_uuid 657ff65a5ac0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:9887, start at 2025-04-04 16:09:53.283441584 +0800 CST m=+3.398628483 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:53.291 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:53.296 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:53.296 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367202340662815 TiDB Cluster ID. tikv_gc_leader_uuid 657ff65a5ac0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:9887, start at 2025-04-04 16:09:53.283441584 +0800 CST m=+3.398628483 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:53.291 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:53.296 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:53.296 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367202836290349 TiDB Cluster ID. tikv_gc_leader_uuid 657ff65a4a40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:9952, start at 2025-04-04 16:09:53.197397074 +0800 CST m=+3.205924071 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:11:53.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:09:53.259 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:59:53.259 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x Command succeeded on attempt 1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3562 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be1e9578-0e8c-49f3-b9b7-6af66e555f06 {"id":"be1e9578-0e8c-49f3-b9b7-6af66e555f06","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754191} /tidb/cdc/default/__cdc_meta__/capture/c84b6c0b-7b6e-413a-8736-9b7a58b23cbc {"id":"c84b6c0b-7b6e-413a-8736-9b7a58b23cbc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754157} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cda1f be1e9578-0e8c-49f3-b9b7-6af66e555f06 /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cda1f be1e9578-0e8c-49f3-b9b7-6af66e555f06 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4490057613477854989,"high":15543667930793430949},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366983529039173,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:10.0536995+08:00","start-ts":457114686664736770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687909920773} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114696403386374,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be1e9578-0e8c-49f3-b9b7-6af66e555f06 {"id":"be1e9578-0e8c-49f3-b9b7-6af66e555f06","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754191} /tidb/cdc/default/__cdc_meta__/capture/c84b6c0b-7b6e-413a-8736-9b7a58b23cbc {"id":"c84b6c0b-7b6e-413a-8736-9b7a58b23cbc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754157} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cda1f be1e9578-0e8c-49f3-b9b7-6af66e555f06 /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cda1f be1e9578-0e8c-49f3-b9b7-6af66e555f06 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4490057613477854989,"high":15543667930793430949},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366983529039173,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:10.0536995+08:00","start-ts":457114686664736770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687909920773} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114696403386374,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be1e9578-0e8c-49f3-b9b7-6af66e555f06 {"id":"be1e9578-0e8c-49f3-b9b7-6af66e555f06","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754191} /tidb/cdc/default/__cdc_meta__/capture/c84b6c0b-7b6e-413a-8736-9b7a58b23cbc {"id":"c84b6c0b-7b6e-413a-8736-9b7a58b23cbc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754157} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd87cda1f be1e9578-0e8c-49f3-b9b7-6af66e555f06 /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cd426 c84b6c0b-7b6e-413a-8736-9b7a58b23cbc /tidb/cdc/default/__cdc_meta__/owner/223195ffd87cda1f be1e9578-0e8c-49f3-b9b7-6af66e555f06 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4490057613477854989,"high":15543667930793430949},"display":{"name":"test","namespace":"default"}},"upstream-id":7489366983529039173,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:10.0536995+08:00","start-ts":457114686664736770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114687909920773} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114696403386374,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_A Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... + set +x + tso='457114699312922626 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114699312922626 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:54 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1572215724.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 not exists for 1-th check, retry later Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/changefeed_pause_resume/stdout.log no DATA RACE found [Fri Apr 4 16:09:52 CST 2025] <<<<<< run test case changefeed_pause_resume success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.cli.13602.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:55 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3562 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93aad4e0-a697-4ba1-bfd4-dbf43daea6ff {"id":"93aad4e0-a697-4ba1-bfd4-dbf43daea6ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754192} /tidb/cdc/default/__cdc_meta__/capture/bd1c5e35-9e7e-4225-a6c6-566d73f72efd {"id":"bd1c5e35-9e7e-4225-a6c6-566d73f72efd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754165} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895f4e6 93aad4e0-a697-4ba1-bfd4-dbf43daea6ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/__cdc_meta__/owner/223195ffd895f4e6 93aad4e0-a697-4ba1-bfd4-dbf43daea6ff /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":813933754614727390,"high":2004519558964326820},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367015043551002,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:16.943277028+08:00","start-ts":457114688467238913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114689712422920} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114698743021572,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93aad4e0-a697-4ba1-bfd4-dbf43daea6ff {"id":"93aad4e0-a697-4ba1-bfd4-dbf43daea6ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754192} /tidb/cdc/default/__cdc_meta__/capture/bd1c5e35-9e7e-4225-a6c6-566d73f72efd {"id":"bd1c5e35-9e7e-4225-a6c6-566d73f72efd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754165} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895f4e6 93aad4e0-a697-4ba1-bfd4-dbf43daea6ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/__cdc_meta__/owner/223195ffd895f4e6 93aad4e0-a697-4ba1-bfd4-dbf43daea6ff /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":813933754614727390,"high":2004519558964326820},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367015043551002,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:16.943277028+08:00","start-ts":457114688467238913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114689712422920} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114698743021572,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93aad4e0-a697-4ba1-bfd4-dbf43daea6ff {"id":"93aad4e0-a697-4ba1-bfd4-dbf43daea6ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754192} /tidb/cdc/default/__cdc_meta__/capture/bd1c5e35-9e7e-4225-a6c6-566d73f72efd {"id":"bd1c5e35-9e7e-4225-a6c6-566d73f72efd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754165} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd895f4e6 93aad4e0-a697-4ba1-bfd4-dbf43daea6ff /tidb/cdc/default/__cdc_meta__/owner/223195ffd895f04e bd1c5e35-9e7e-4225-a6c6-566d73f72efd /tidb/cdc/default/__cdc_meta__/owner/223195ffd895f4e6 93aad4e0-a697-4ba1-bfd4-dbf43daea6ff /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":813933754614727390,"high":2004519558964326820},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367015043551002,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:16.943277028+08:00","start-ts":457114688467238913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114689712422920} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114698743021572,"progress":1}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:55 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee2baaee-4419-4f0c-b2c4-ba9c21205292 {"id":"ee2baaee-4419-4f0c-b2c4-ba9c21205292","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754192} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e0e93c ee2baaee-4419-4f0c-b2c4-ba9c21205292 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e0e93c ee2baaee-4419-4f0c-b2c4-ba9c21205292' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee2baaee-4419-4f0c-b2c4-ba9c21205292 {"id":"ee2baaee-4419-4f0c-b2c4-ba9c21205292","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754192} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e0e93c ee2baaee-4419-4f0c-b2c4-ba9c21205292 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e0e93c ee2baaee-4419-4f0c-b2c4-ba9c21205292' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee2baaee-4419-4f0c-b2c4-ba9c21205292 {"id":"ee2baaee-4419-4f0c-b2c4-ba9c21205292","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754192} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e0e93c ee2baaee-4419-4f0c-b2c4-ba9c21205292 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e0e93c ee2baaee-4419-4f0c-b2c4-ba9c21205292' + grep -q 'etcd info' + break + set +x 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:55 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfb96ae0-a73e-49b0-af6c-396ec924075a {"id":"bfb96ae0-a73e-49b0-af6c-396ec924075a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754193} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfb96ae0-a73e-49b0-af6c-396ec924075a {"id":"bfb96ae0-a73e-49b0-af6c-396ec924075a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754193} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfb96ae0-a73e-49b0-af6c-396ec924075a {"id":"bfb96ae0-a73e-49b0-af6c-396ec924075a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754193} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114698736205825 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.9643.out cli changefeed create --start-ts=457114698736205825 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367178140310957,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:56.110054484+08:00","start_ts":457114698736205825,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114698736205825,"checkpoint_ts":457114698736205825,"checkpoint_time":"2025-04-04 16:09:51.346","gid":{"low":9750020843988387933,"high":8751326180401707658}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 {"id":"7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754190} /tidb/cdc/default/__cdc_meta__/capture/f1f05caf-18bc-4fb4-9671-d204ad352d2d {"id":"f1f05caf-18bc-4fb4-9671-d204ad352d2d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754193} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a0600b f1f05caf-18bc-4fb4-9671-d204ad352d2d /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a0600b f1f05caf-18bc-4fb4-9671-d204ad352d2d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114697751592962,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 {"id":"7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754190} /tidb/cdc/default/__cdc_meta__/capture/f1f05caf-18bc-4fb4-9671-d204ad352d2d {"id":"f1f05caf-18bc-4fb4-9671-d204ad352d2d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754193} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a0600b f1f05caf-18bc-4fb4-9671-d204ad352d2d /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a0600b f1f05caf-18bc-4fb4-9671-d204ad352d2d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114697751592962,"progress":1}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 {"id":"7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754190} /tidb/cdc/default/__cdc_meta__/capture/f1f05caf-18bc-4fb4-9671-d204ad352d2d {"id":"f1f05caf-18bc-4fb4-9671-d204ad352d2d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754193} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05ac3 b2139680-8548-44b3-bfbb-31d63ec8206e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8a0600b f1f05caf-18bc-4fb4-9671-d204ad352d2d /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a05fa5 7cb72c5b-58c0-4406-ae6c-1ed4a5c8fda1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8a0600b f1f05caf-18bc-4fb4-9671-d204ad352d2d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6002285263978123182,"high":14315543091948369078},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367024372905433,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:19.178711511+08:00","start-ts":457114689048674305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114690293858313} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114697751592962,"progress":1}' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.11151.out cli tso query --pd=http://127.0.0.1:2379 table fail_over_ddl_test.test2 exists + set +x + tso='457114699837997057 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114699837997057 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:56 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1362913631.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (12875) - No such process wait process 12875 exit for 1-th time... process 12875 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:09:57 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1300813010.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later + set +x + tso='457114700151783425 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114700151783425 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:09:58 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.1118411186.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:57 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43c6a826-58f0-46ea-8ea4-19fcb64e16ac {"id":"43c6a826-58f0-46ea-8ea4-19fcb64e16ac","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754195} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43c6a826-58f0-46ea-8ea4-19fcb64e16ac {"id":"43c6a826-58f0-46ea-8ea4-19fcb64e16ac","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754195} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43c6a826-58f0-46ea-8ea4-19fcb64e16ac {"id":"43c6a826-58f0-46ea-8ea4-19fcb64e16ac","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754195} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114699312922626 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.15786.out cli changefeed create --start-ts=457114699312922626 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367190626617227,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:09:58.343823272+08:00","start_ts":457114699312922626,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114699312922626,"checkpoint_ts":457114699312922626,"checkpoint_time":"2025-04-04 16:09:53.546","gid":{"low":10180017735251369316,"high":2523931557934127279}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCase-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory check diff failed 2-th time, retry later shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (9585) - No such process wait process 9585 exit for 1-th time... process 9585 already exit [Fri Apr 4 16:10:00 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.97269728.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:09:59 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64db1f66-499f-4b3d-a3e6-706bd7621172 {"id":"64db1f66-499f-4b3d-a3e6-706bd7621172","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64db1f66-499f-4b3d-a3e6-706bd7621172 {"id":"64db1f66-499f-4b3d-a3e6-706bd7621172","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64db1f66-499f-4b3d-a3e6-706bd7621172 {"id":"64db1f66-499f-4b3d-a3e6-706bd7621172","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114699837997057 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.cli.13686.out cli changefeed create --start-ts=457114699837997057 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367200255130049,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:00.304868896+08:00","start_ts":457114699837997057,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114699837997057,"checkpoint_ts":457114699837997057,"checkpoint_time":"2025-04-04 16:09:55.549","gid":{"low":5569667663057295696,"high":15486335412544235427}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10596) - No such process wait process 10596 exit for 1-th time... process 10596 already exit [Fri Apr 4 16:09:58 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1077210774.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3083 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6cf78b2-688f-48ef-bd4e-120331c91635 {"id":"c6cf78b2-688f-48ef-bd4e-120331c91635","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5065050923354093612,"high":7008916332082200506},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367166296562395,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:53.5448803+08:00","start-ts":457114698071408642,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699303485451} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114699683594267,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6cf78b2-688f-48ef-bd4e-120331c91635 {"id":"c6cf78b2-688f-48ef-bd4e-120331c91635","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5065050923354093612,"high":7008916332082200506},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367166296562395,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:53.5448803+08:00","start-ts":457114698071408642,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699303485451} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114699683594267,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6cf78b2-688f-48ef-bd4e-120331c91635 {"id":"c6cf78b2-688f-48ef-bd4e-120331c91635","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5065050923354093612,"high":7008916332082200506},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367166296562395,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:53.5448803+08:00","start-ts":457114698071408642,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699303485451} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114699683594267,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:00 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1308013082.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully command: check_safepoint_forward http://127.0.0.1:2379 7489367088931747056 457114699174772735 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10612) - No such process wait process 10612 exit for 1-th time... process 10612 already exit [Fri Apr 4 16:09:58 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1080510807.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseD-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:01 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43fab308-c4bf-4ec5-a4a7-5eb3869aad42 {"id":"43fab308-c4bf-4ec5-a4a7-5eb3869aad42","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754198} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42 /tidb/cdc/default/__cdc_meta__/owner/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43fab308-c4bf-4ec5-a4a7-5eb3869aad42 {"id":"43fab308-c4bf-4ec5-a4a7-5eb3869aad42","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754198} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42 /tidb/cdc/default/__cdc_meta__/owner/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43fab308-c4bf-4ec5-a4a7-5eb3869aad42 {"id":"43fab308-c4bf-4ec5-a4a7-5eb3869aad42","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754198} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42 /tidb/cdc/default/__cdc_meta__/owner/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114700151783425 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.11250.out cli changefeed create --start-ts=457114700151783425 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367202340662815,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:01.515468822+08:00","start_ts":457114700151783425,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114700151783425,"checkpoint_ts":457114700151783425,"checkpoint_time":"2025-04-04 16:09:56.746","gid":{"low":7658969282484924454,"high":9210270466288278959}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x Command succeeded on attempt 1 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseI-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table fail_over_ddl_test.test2 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46c9cdef-cbdc-4689-817c-f5e1ea80f444 {"id":"46c9cdef-cbdc-4689-817c-f5e1ea80f444","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754172} /tidb/cdc/default/__cdc_meta__/capture/a30ee4a4-220a-460a-a98a-200298a1a2ea {"id":"a30ee4a4-220a-460a-a98a-200298a1a2ea","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754199} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd67b4 a30ee4a4-220a-460a-a98a-200298a1a2ea /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd67b4 a30ee4a4-220a-460a-a98a-200298a1a2ea /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4343697654294407245,"high":14354032950356581550},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367051100265341,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.597229711+08:00","start-ts":457114690996666369,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692242112520} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695492698120,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46c9cdef-cbdc-4689-817c-f5e1ea80f444 {"id":"46c9cdef-cbdc-4689-817c-f5e1ea80f444","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754172} /tidb/cdc/default/__cdc_meta__/capture/a30ee4a4-220a-460a-a98a-200298a1a2ea {"id":"a30ee4a4-220a-460a-a98a-200298a1a2ea","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754199} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd67b4 a30ee4a4-220a-460a-a98a-200298a1a2ea /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd67b4 a30ee4a4-220a-460a-a98a-200298a1a2ea /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4343697654294407245,"high":14354032950356581550},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367051100265341,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.597229711+08:00","start-ts":457114690996666369,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692242112520} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695492698120,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46c9cdef-cbdc-4689-817c-f5e1ea80f444 {"id":"46c9cdef-cbdc-4689-817c-f5e1ea80f444","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754172} /tidb/cdc/default/__cdc_meta__/capture/a30ee4a4-220a-460a-a98a-200298a1a2ea {"id":"a30ee4a4-220a-460a-a98a-200298a1a2ea","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754199} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8bd67b4 a30ee4a4-220a-460a-a98a-200298a1a2ea /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd62e1 46c9cdef-cbdc-4689-817c-f5e1ea80f444 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8bd67b4 a30ee4a4-220a-460a-a98a-200298a1a2ea /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4343697654294407245,"high":14354032950356581550},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367051100265341,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.597229711+08:00","start-ts":457114690996666369,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692242112520} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114695492698120,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/805958b2-c48d-492e-b615-06fac12de406 {"id":"805958b2-c48d-492e-b615-06fac12de406","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754199} /tidb/cdc/default/__cdc_meta__/capture/adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 {"id":"adbdf3ff-3990-4a28-8d70-e0f62c4d89f9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754171} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c18eed 805958b2-c48d-492e-b615-06fac12de406 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c18eed 805958b2-c48d-492e-b615-06fac12de406 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7370088951958399152,"high":13055422429210467463},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367053141610012,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.609050952+08:00","start-ts":457114691000336385,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692245520391} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700503056389,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/805958b2-c48d-492e-b615-06fac12de406 {"id":"805958b2-c48d-492e-b615-06fac12de406","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754199} /tidb/cdc/default/__cdc_meta__/capture/adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 {"id":"adbdf3ff-3990-4a28-8d70-e0f62c4d89f9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754171} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c18eed 805958b2-c48d-492e-b615-06fac12de406 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c18eed 805958b2-c48d-492e-b615-06fac12de406 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7370088951958399152,"high":13055422429210467463},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367053141610012,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.609050952+08:00","start-ts":457114691000336385,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692245520391} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700503056389,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/805958b2-c48d-492e-b615-06fac12de406 {"id":"805958b2-c48d-492e-b615-06fac12de406","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754199} /tidb/cdc/default/__cdc_meta__/capture/adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 {"id":"adbdf3ff-3990-4a28-8d70-e0f62c4d89f9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754171} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8c18eed 805958b2-c48d-492e-b615-06fac12de406 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c189c4 adbdf3ff-3990-4a28-8d70-e0f62c4d89f9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8c18eed 805958b2-c48d-492e-b615-06fac12de406 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7370088951958399152,"high":13055422429210467463},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367053141610012,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:26.609050952+08:00","start-ts":457114691000336385,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114692245520391} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700503056389,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later safepoint 457114699174772735 is not forward run task failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15727) - No such process wait process 15727 exit for 1-th time... process 15727 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:10:02 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1585315855.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Command succeeded on attempt 1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9a69157-9fd6-4eb7-ac1f-8754232b5d4a {"id":"b9a69157-9fd6-4eb7-ac1f-8754232b5d4a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754200} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9750020843988387933,"high":8751326180401707658},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367178140310957,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:56.110054484+08:00","start-ts":457114698736205825,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699981389829} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700361498631,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9a69157-9fd6-4eb7-ac1f-8754232b5d4a {"id":"b9a69157-9fd6-4eb7-ac1f-8754232b5d4a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754200} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9750020843988387933,"high":8751326180401707658},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367178140310957,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:56.110054484+08:00","start-ts":457114698736205825,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699981389829} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700361498631,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9a69157-9fd6-4eb7-ac1f-8754232b5d4a {"id":"b9a69157-9fd6-4eb7-ac1f-8754232b5d4a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754200} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9750020843988387933,"high":8751326180401707658},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367178140310957,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:56.110054484+08:00","start-ts":457114698736205825,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699981389829} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700361498631,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:03 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=1*sleep(30000)->return(false)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.98119813.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367243016454897 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6634f8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:11669, start at 2025-04-04 16:10:02.453754414 +0800 CST m=+3.706659700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:02.462 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:02.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:02.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367243016454897 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6634f8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:11669, start at 2025-04-04 16:10:02.453754414 +0800 CST m=+3.706659700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:02.462 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:02.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:02.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367240521552151 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6636880004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:11735, start at 2025-04-04 16:10:02.535204837 +0800 CST m=+3.670017740 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:02.544 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:02.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:02.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table fail_over_ddl_test.test2 not exists for 1-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/fail_over_ddl_B/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3664 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79c709c4-0670-4d49-b505-ecfada374ad1 {"id":"79c709c4-0670-4d49-b505-ecfada374ad1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754201} /tidb/cdc/default/__cdc_meta__/capture/c6cf78b2-688f-48ef-bd4e-120331c91635 {"id":"c6cf78b2-688f-48ef-bd4e-120331c91635","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cb25 79c709c4-0670-4d49-b505-ecfada374ad1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cb25 79c709c4-0670-4d49-b505-ecfada374ad1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5065050923354093612,"high":7008916332082200506},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367166296562395,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:53.5448803+08:00","start-ts":457114698071408642,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699303485451} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701793853447,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79c709c4-0670-4d49-b505-ecfada374ad1 {"id":"79c709c4-0670-4d49-b505-ecfada374ad1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754201} /tidb/cdc/default/__cdc_meta__/capture/c6cf78b2-688f-48ef-bd4e-120331c91635 {"id":"c6cf78b2-688f-48ef-bd4e-120331c91635","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cb25 79c709c4-0670-4d49-b505-ecfada374ad1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cb25 79c709c4-0670-4d49-b505-ecfada374ad1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5065050923354093612,"high":7008916332082200506},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367166296562395,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:53.5448803+08:00","start-ts":457114698071408642,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699303485451} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701793853447,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79c709c4-0670-4d49-b505-ecfada374ad1 {"id":"79c709c4-0670-4d49-b505-ecfada374ad1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754201} /tidb/cdc/default/__cdc_meta__/capture/c6cf78b2-688f-48ef-bd4e-120331c91635 {"id":"c6cf78b2-688f-48ef-bd4e-120331c91635","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926c9e7 254bedfe-dcd6-42e7-a4b5-ccd9f7db7690 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cb25 79c709c4-0670-4d49-b505-ecfada374ad1 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cb25 79c709c4-0670-4d49-b505-ecfada374ad1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5065050923354093612,"high":7008916332082200506},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367166296562395,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:53.5448803+08:00","start-ts":457114698071408642,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699303485451} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701793853447,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.13149.out cli capture list check diff failed 2-th time, retry later command: check_safepoint_forward http://127.0.0.1:2379 7489367088931747056 457114699174772735 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 4-th time, retry later table fail_over_ddl_test.test2 not exists for 1-th check, retry later safepoint 457114699174772735 is not forward run task failed 2-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.cli.13089.out cli tso query --pd=http://127.0.0.1:2379 + set +x + ans='[ { "id": "c6cf78b2-688f-48ef-bd4e-120331c91635", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "79c709c4-0670-4d49-b505-ecfada374ad1", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"c6cf78b2-688f-48ef-bd4e-120331c91635",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"79c709c4-0670-4d49-b505-ecfada374ad1",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=79c709c4-0670-4d49-b505-ecfada374ad1 + '[' -z 79c709c4-0670-4d49-b505-ecfada374ad1 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 79c709c4-0670-4d49-b505-ecfada374ad1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.13180.out cli changefeed move-table -c test -t 114 -d 79c709c4-0670-4d49-b505-ecfada374ad1 table fail_over_ddl_test.test2 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8dfb8894-2ecd-4813-b06d-0ac1bf49cdef {"id":"8dfb8894-2ecd-4813-b06d-0ac1bf49cdef","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754202} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10180017735251369316,"high":2523931557934127279},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367190626617227,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:58.343823272+08:00","start-ts":457114699312922626,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114700571475971} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700938477569,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8dfb8894-2ecd-4813-b06d-0ac1bf49cdef {"id":"8dfb8894-2ecd-4813-b06d-0ac1bf49cdef","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754202} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10180017735251369316,"high":2523931557934127279},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367190626617227,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:58.343823272+08:00","start-ts":457114699312922626,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114700571475971} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700938477569,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8dfb8894-2ecd-4813-b06d-0ac1bf49cdef {"id":"8dfb8894-2ecd-4813-b06d-0ac1bf49cdef","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754202} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10180017735251369316,"high":2523931557934127279},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367190626617227,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:58.343823272+08:00","start-ts":457114699312922626,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114700571475971} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700938477569,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:05 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1592815930.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13634) - No such process wait process 13634 exit for 1-th time... process 13634 already exit [Fri Apr 4 16:10:05 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1376213764.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_B Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9a69157-9fd6-4eb7-ac1f-8754232b5d4a {"id":"b9a69157-9fd6-4eb7-ac1f-8754232b5d4a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754200} /tidb/cdc/default/__cdc_meta__/capture/e5f189eb-6f3b-4e10-b8a2-088ffb019a75 {"id":"e5f189eb-6f3b-4e10-b8a2-088ffb019a75","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754203} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9750020843988387933,"high":8751326180401707658},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367178140310957,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:56.110054484+08:00","start-ts":457114698736205825,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699981389829} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700361498649,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9a69157-9fd6-4eb7-ac1f-8754232b5d4a {"id":"b9a69157-9fd6-4eb7-ac1f-8754232b5d4a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754200} /tidb/cdc/default/__cdc_meta__/capture/e5f189eb-6f3b-4e10-b8a2-088ffb019a75 {"id":"e5f189eb-6f3b-4e10-b8a2-088ffb019a75","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754203} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9750020843988387933,"high":8751326180401707658},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367178140310957,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:56.110054484+08:00","start-ts":457114698736205825,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699981389829} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700361498649,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9a69157-9fd6-4eb7-ac1f-8754232b5d4a {"id":"b9a69157-9fd6-4eb7-ac1f-8754232b5d4a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754200} /tidb/cdc/default/__cdc_meta__/capture/e5f189eb-6f3b-4e10-b8a2-088ffb019a75 {"id":"e5f189eb-6f3b-4e10-b8a2-088ffb019a75","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754203} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b5e8 bfb96ae0-a73e-49b0-af6c-396ec924075a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b6c5 b9a69157-9fd6-4eb7-ac1f-8754232b5d4a /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9750020843988387933,"high":8751326180401707658},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367178140310957,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:56.110054484+08:00","start-ts":457114698736205825,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699981389829} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700361498649,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.9892.out cli capture list table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (11189) - No such process wait process 11189 exit for 1-th time... process 11189 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:10:07 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.1133611338.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114702506885121 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114702506885121 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:07 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1311613118.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_D Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_E Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x + ans='[ { "id": "b9a69157-9fd6-4eb7-ac1f-8754232b5d4a", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "e5f189eb-6f3b-4e10-b8a2-088ffb019a75", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"b9a69157-9fd6-4eb7-ac1f-8754232b5d4a",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"e5f189eb-6f3b-4e10-b8a2-088ffb019a75",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=e5f189eb-6f3b-4e10-b8a2-088ffb019a75 + '[' -z e5f189eb-6f3b-4e10-b8a2-088ffb019a75 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d e5f189eb-6f3b-4e10-b8a2-088ffb019a75 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.9923.out cli changefeed move-table -c test -t 114 -d e5f189eb-6f3b-4e10-b8a2-088ffb019a75 command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 4-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8dfb8894-2ecd-4813-b06d-0ac1bf49cdef {"id":"8dfb8894-2ecd-4813-b06d-0ac1bf49cdef","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754202} /tidb/cdc/default/__cdc_meta__/capture/c1751024-4251-4853-9ca5-d4df0ad6d22c {"id":"c1751024-4251-4853-9ca5-d4df0ad6d22c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754205} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394d29 c1751024-4251-4853-9ca5-d4df0ad6d22c /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394d29 c1751024-4251-4853-9ca5-d4df0ad6d22c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10180017735251369316,"high":2523931557934127279},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367190626617227,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:58.343823272+08:00","start-ts":457114699312922626,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114700571475971} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700938477569,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8dfb8894-2ecd-4813-b06d-0ac1bf49cdef {"id":"8dfb8894-2ecd-4813-b06d-0ac1bf49cdef","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754202} /tidb/cdc/default/__cdc_meta__/capture/c1751024-4251-4853-9ca5-d4df0ad6d22c {"id":"c1751024-4251-4853-9ca5-d4df0ad6d22c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754205} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394d29 c1751024-4251-4853-9ca5-d4df0ad6d22c /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394d29 c1751024-4251-4853-9ca5-d4df0ad6d22c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10180017735251369316,"high":2523931557934127279},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367190626617227,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:58.343823272+08:00","start-ts":457114699312922626,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114700571475971} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700938477569,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8dfb8894-2ecd-4813-b06d-0ac1bf49cdef {"id":"8dfb8894-2ecd-4813-b06d-0ac1bf49cdef","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754202} /tidb/cdc/default/__cdc_meta__/capture/c1751024-4251-4853-9ca5-d4df0ad6d22c {"id":"c1751024-4251-4853-9ca5-d4df0ad6d22c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754205} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394be3 43c6a826-58f0-46ea-8ea4-19fcb64e16ac /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394d29 c1751024-4251-4853-9ca5-d4df0ad6d22c /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394d29 c1751024-4251-4853-9ca5-d4df0ad6d22c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10180017735251369316,"high":2523931557934127279},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367190626617227,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:58.343823272+08:00","start-ts":457114699312922626,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114700571475971} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114700938477569,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.16003.out cli capture list chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_I Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 {"id":"8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754206} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701620314133,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 {"id":"8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754206} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701620314133,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 {"id":"8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754206} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701620314133,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:10:08 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1383513837.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 command: check_safepoint_forward http://127.0.0.1:2379 7489367088931747056 457114699174772735 Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24820) - No such process wait process 24820 exit for 1-th time... process 24820 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24893) - No such process wait process 24893 exit for 1-th time... process 24893 already exit [Fri Apr 4 16:10:07 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.2506825070.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 safepoint 457114699174772735 is not forward run task failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:10 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ce4c062c-53ec-46d7-a1e5-715630335553 {"id":"ce4c062c-53ec-46d7-a1e5-715630335553","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/__cdc_meta__/owner/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7658969282484924454,"high":9210270466288278959},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367202340662815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:01.515468822+08:00","start-ts":457114700151783425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701397229575} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701908410383,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ce4c062c-53ec-46d7-a1e5-715630335553 {"id":"ce4c062c-53ec-46d7-a1e5-715630335553","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/__cdc_meta__/owner/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7658969282484924454,"high":9210270466288278959},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367202340662815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:01.515468822+08:00","start-ts":457114700151783425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701397229575} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701908410383,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ce4c062c-53ec-46d7-a1e5-715630335553 {"id":"ce4c062c-53ec-46d7-a1e5-715630335553","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/__cdc_meta__/owner/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7658969282484924454,"high":9210270466288278959},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367202340662815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:01.515468822+08:00","start-ts":457114700151783425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701397229575} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701908410383,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:10 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.1142011422.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + ans='[ { "id": "8dfb8894-2ecd-4813-b06d-0ac1bf49cdef", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "c1751024-4251-4853-9ca5-d4df0ad6d22c", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"8dfb8894-2ecd-4813-b06d-0ac1bf49cdef",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"c1751024-4251-4853-9ca5-d4df0ad6d22c",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=c1751024-4251-4853-9ca5-d4df0ad6d22c + '[' -z c1751024-4251-4853-9ca5-d4df0ad6d22c ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d c1751024-4251-4853-9ca5-d4df0ad6d22c + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.16034.out cli changefeed move-table -c test -t 114 -d c1751024-4251-4853-9ca5-d4df0ad6d22c Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:10 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e24468e-5caa-4d85-9356-41c79eaabd84 {"id":"4e24468e-5caa-4d85-9356-41c79eaabd84","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84 /tidb/cdc/default/__cdc_meta__/owner/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e24468e-5caa-4d85-9356-41c79eaabd84 {"id":"4e24468e-5caa-4d85-9356-41c79eaabd84","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84 /tidb/cdc/default/__cdc_meta__/owner/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e24468e-5caa-4d85-9356-41c79eaabd84 {"id":"4e24468e-5caa-4d85-9356-41c79eaabd84","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84 /tidb/cdc/default/__cdc_meta__/owner/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114702506885121 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.cli.13191.out cli changefeed create --start-ts=457114702506885121 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367243016454897,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:10.52520532+08:00","start_ts":457114702506885121,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114702506885121,"checkpoint_ts":457114702506885121,"checkpoint_time":"2025-04-04 16:10:05.730","gid":{"low":2903904039547186801,"high":14969412283366187181}} PASS command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:10 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b278baae-203c-4415-99ab-7768e53492ac {"id":"b278baae-203c-4415-99ab-7768e53492ac","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114698703962113,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b278baae-203c-4415-99ab-7768e53492ac {"id":"b278baae-203c-4415-99ab-7768e53492ac","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114698703962113,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b278baae-203c-4415-99ab-7768e53492ac {"id":"b278baae-203c-4415-99ab-7768e53492ac","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114698703962113,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:10 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.2513925141.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.7% of statements in github.com/pingcap/ticdc/... check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=3 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseL-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (11106) - No such process wait process 11106 exit for 1-th time... process 11106 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (11164) - No such process wait process 11164 exit for 1-th time... process 11164 already exit [Fri Apr 4 16:10:08 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1129911301.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33d78d48-85db-4d4b-a008-076d16e37dff {"id":"33d78d48-85db-4d4b-a008-076d16e37dff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754209} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114702302937101,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33d78d48-85db-4d4b-a008-076d16e37dff {"id":"33d78d48-85db-4d4b-a008-076d16e37dff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754209} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114702302937101,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33d78d48-85db-4d4b-a008-076d16e37dff {"id":"33d78d48-85db-4d4b-a008-076d16e37dff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754209} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114702302937101,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":2}' + echo '{"count":2}' {"count":2} ++ echo '{"count":2}' ++ jq -r .count + value=2 + '[' -1 == -1 ']' + '[' 2 '!=' null ']' + exit 0 [Fri Apr 4 16:10:11 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1136411366.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Command succeeded on attempt 1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/691c6de6-6bd1-42c0-b805-7c8c2560d44d {"id":"691c6de6-6bd1-42c0-b805-7c8c2560d44d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754209} /tidb/cdc/default/__cdc_meta__/capture/8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 {"id":"8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754206} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f5552 691c6de6-6bd1-42c0-b805-7c8c2560d44d /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f5552 691c6de6-6bd1-42c0-b805-7c8c2560d44d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701620314133,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/691c6de6-6bd1-42c0-b805-7c8c2560d44d {"id":"691c6de6-6bd1-42c0-b805-7c8c2560d44d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754209} /tidb/cdc/default/__cdc_meta__/capture/8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 {"id":"8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754206} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f5552 691c6de6-6bd1-42c0-b805-7c8c2560d44d /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f5552 691c6de6-6bd1-42c0-b805-7c8c2560d44d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701620314133,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/691c6de6-6bd1-42c0-b805-7c8c2560d44d {"id":"691c6de6-6bd1-42c0-b805-7c8c2560d44d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754209} /tidb/cdc/default/__cdc_meta__/capture/8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 {"id":"8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754206} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f53e4 64db1f66-499f-4b3d-a3e6-706bd7621172 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f5552 691c6de6-6bd1-42c0-b805-7c8c2560d44d /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f5552 691c6de6-6bd1-42c0-b805-7c8c2560d44d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701620314133,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8921c6c6-7121-427c-a7da-4d9d9a4f2c66 {"id":"8921c6c6-7121-427c-a7da-4d9d9a4f2c66","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754210} /tidb/cdc/default/__cdc_meta__/capture/ce4c062c-53ec-46d7-a1e5-715630335553 {"id":"ce4c062c-53ec-46d7-a1e5-715630335553","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/__cdc_meta__/owner/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7658969282484924454,"high":9210270466288278959},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367202340662815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:01.515468822+08:00","start-ts":457114700151783425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701397229575} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701908410383,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8921c6c6-7121-427c-a7da-4d9d9a4f2c66 {"id":"8921c6c6-7121-427c-a7da-4d9d9a4f2c66","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754210} /tidb/cdc/default/__cdc_meta__/capture/ce4c062c-53ec-46d7-a1e5-715630335553 {"id":"ce4c062c-53ec-46d7-a1e5-715630335553","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/__cdc_meta__/owner/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7658969282484924454,"high":9210270466288278959},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367202340662815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:01.515468822+08:00","start-ts":457114700151783425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701397229575} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701908410383,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8921c6c6-7121-427c-a7da-4d9d9a4f2c66 {"id":"8921c6c6-7121-427c-a7da-4d9d9a4f2c66","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754210} /tidb/cdc/default/__cdc_meta__/capture/ce4c062c-53ec-46d7-a1e5-715630335553 {"id":"ce4c062c-53ec-46d7-a1e5-715630335553","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94631e5 43fab308-c4bf-4ec5-a4a7-5eb3869aad42 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/__cdc_meta__/owner/223195ffd94632ea ce4c062c-53ec-46d7-a1e5-715630335553 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7658969282484924454,"high":9210270466288278959},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367202340662815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:01.515468822+08:00","start-ts":457114700151783425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701397229575} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114701908410383,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.11494.out cli capture list Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fdff261-b2f1-4afc-9a33-abf5a9222073 {"id":"7fdff261-b2f1-4afc-9a33-abf5a9222073","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754211} /tidb/cdc/default/__cdc_meta__/capture/b278baae-203c-4415-99ab-7768e53492ac {"id":"b278baae-203c-4415-99ab-7768e53492ac","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53fea 7fdff261-b2f1-4afc-9a33-abf5a9222073 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53fea 7fdff261-b2f1-4afc-9a33-abf5a9222073 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114698703962113,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fdff261-b2f1-4afc-9a33-abf5a9222073 {"id":"7fdff261-b2f1-4afc-9a33-abf5a9222073","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754211} /tidb/cdc/default/__cdc_meta__/capture/b278baae-203c-4415-99ab-7768e53492ac {"id":"b278baae-203c-4415-99ab-7768e53492ac","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53fea 7fdff261-b2f1-4afc-9a33-abf5a9222073 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53fea 7fdff261-b2f1-4afc-9a33-abf5a9222073 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114698703962113,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fdff261-b2f1-4afc-9a33-abf5a9222073 {"id":"7fdff261-b2f1-4afc-9a33-abf5a9222073","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754211} /tidb/cdc/default/__cdc_meta__/capture/b278baae-203c-4415-99ab-7768e53492ac {"id":"b278baae-203c-4415-99ab-7768e53492ac","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754207} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53ac5 dad8d493-223f-46a3-b155-11fcbcf08790 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8e53fea 7fdff261-b2f1-4afc-9a33-abf5a9222073 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53f82 b278baae-203c-4415-99ab-7768e53492ac /tidb/cdc/default/__cdc_meta__/owner/223195ffd8e53fea 7fdff261-b2f1-4afc-9a33-abf5a9222073 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8380935837768221898,"high":1525368754776906891},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367092301519374,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:36.905507251+08:00","start-ts":457114693697273857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114694942195722} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114698703962113,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully check diff failed 1-th time, retry later table fail_over_ddl_test.test2 not exists for 1-th check, retry later check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseO-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later + set +x + ans='[ { "id": "8921c6c6-7121-427c-a7da-4d9d9a4f2c66", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" }, { "id": "ce4c062c-53ec-46d7-a1e5-715630335553", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"8921c6c6-7121-427c-a7da-4d9d9a4f2c66",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '},' '{' '"id":' '"ce4c062c-53ec-46d7-a1e5-715630335553",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=8921c6c6-7121-427c-a7da-4d9d9a4f2c66 + '[' -z 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.11525.out cli changefeed move-table -c test -t 114 -d 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13767) - No such process wait process 13767 exit for 1-th time... process 13767 already exit + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33d78d48-85db-4d4b-a008-076d16e37dff {"id":"33d78d48-85db-4d4b-a008-076d16e37dff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754209} /tidb/cdc/default/__cdc_meta__/capture/39021e86-c33a-4346-8276-3bd8750cec32 {"id":"39021e86-c33a-4346-8276-3bd8750cec32","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754212} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec31c3 39021e86-c33a-4346-8276-3bd8750cec32 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec31c3 39021e86-c33a-4346-8276-3bd8750cec32 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114702302937101,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33d78d48-85db-4d4b-a008-076d16e37dff {"id":"33d78d48-85db-4d4b-a008-076d16e37dff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754209} /tidb/cdc/default/__cdc_meta__/capture/39021e86-c33a-4346-8276-3bd8750cec32 {"id":"39021e86-c33a-4346-8276-3bd8750cec32","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754212} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec31c3 39021e86-c33a-4346-8276-3bd8750cec32 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec31c3 39021e86-c33a-4346-8276-3bd8750cec32 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test + grep -q 'failed to get info:' {"checkpoint-ts":457114702302937101,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33d78d48-85db-4d4b-a008-076d16e37dff {"id":"33d78d48-85db-4d4b-a008-076d16e37dff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754209} /tidb/cdc/default/__cdc_meta__/capture/39021e86-c33a-4346-8276-3bd8750cec32 {"id":"39021e86-c33a-4346-8276-3bd8750cec32","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754212} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec2cf2 ca3d1581-3f09-42c5-ac59-67d87237b63c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd8ec31c3 39021e86-c33a-4346-8276-3bd8750cec32 /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec3160 33d78d48-85db-4d4b-a008-076d16e37dff /tidb/cdc/default/__cdc_meta__/owner/223195ffd8ec31c3 39021e86-c33a-4346-8276-3bd8750cec32 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":11043631044976089295,"high":5444269735529934263},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367100736289111,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:38.585896555+08:00","start-ts":457114694136889345,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114695382073352} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114702302937101,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15933) - No such process wait process 15933 exit for 1-th time... process 15933 already exit [Fri Apr 4 16:10:15 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1609316095.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13840) - No such process wait process 13840 exit for 1-th time... process 13840 already exit [Fri Apr 4 16:10:15 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1393413936.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13121) - No such process wait process 13121 exit for 1-th time... process 13121 already exit [Fri Apr 4 16:10:16 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1325913261.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367299086423050 TiDB Cluster ID. tikv_gc_leader_uuid 657ff66fea4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:11067, start at 2025-04-04 16:10:15.359300816 +0800 CST m=+3.539089701 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:15.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:15.416 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:15.416 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367299086423050 TiDB Cluster ID. tikv_gc_leader_uuid 657ff66fea4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:11067, start at 2025-04-04 16:10:15.359300816 +0800 CST m=+3.539089701 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:15.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:15.416 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:15.416 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367294400579685 TiDB Cluster ID. tikv_gc_leader_uuid 657ff66feb4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:11153, start at 2025-04-04 16:10:15.3593382 +0800 CST m=+3.417156826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:15.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:15.416 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:15.416 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later check diff failed 1-th time, retry later command: check_safepoint_forward http://127.0.0.1:2379 7489367088931747056 457114699174772735 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCase-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory safepoint 457114699174772735 is not forward run task failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367301609395499 TiDB Cluster ID. tikv_gc_leader_uuid 657ff670a48000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:14595, start at 2025-04-04 16:10:16.103932666 +0800 CST m=+3.297580547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:16.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:16.118 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:16.118 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367301609395499 TiDB Cluster ID. tikv_gc_leader_uuid 657ff670a48000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:14595, start at 2025-04-04 16:10:16.103932666 +0800 CST m=+3.297580547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:16.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:16.118 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:16.118 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367297807600967 TiDB Cluster ID. tikv_gc_leader_uuid 657ff670d30000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:14669, start at 2025-04-04 16:10:16.302948254 +0800 CST m=+3.383396074 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:16.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:16.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:16.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/error.log shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367305182583740 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6712ec0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:11823, start at 2025-04-04 16:10:16.635562784 +0800 CST m=+3.212001417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:16.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:16.700 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:16.700 +0800 All versions after safe point can be accessed. (DO NOT EDIT) arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367305182583740 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6712ec0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:11823, start at 2025-04-04 16:10:16.635562784 +0800 CST m=+3.212001417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:16.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:16.700 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:16.700 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367307847068653 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6711540004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:11902, start at 2025-04-04 16:10:16.537217609 +0800 CST m=+2.996146777 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:16.543 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:16.600 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:16.600 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367304856166827 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6713200004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:12276, start at 2025-04-04 16:10:16.653518418 +0800 CST m=+3.334853650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:16.661 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:16.648 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:16.648 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367304856166827 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6713200004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:12276, start at 2025-04-04 16:10:16.653518418 +0800 CST m=+3.334853650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:16.661 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:16.648 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:16.648 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367294279001803 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6717d8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:12363, start at 2025-04-04 16:10:16.987793617 +0800 CST m=+3.513306486 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:16.995 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:17.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:17.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseM-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.12366.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2fbcc669-f80f-4096-97e2-242b817b2b95 {"id":"2fbcc669-f80f-4096-97e2-242b817b2b95","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754215} /tidb/cdc/default/__cdc_meta__/capture/8dfb8894-2ecd-4813-b06d-0ac1bf49cdef {"id":"8dfb8894-2ecd-4813-b06d-0ac1bf49cdef","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754202} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394f0d 2fbcc669-f80f-4096-97e2-242b817b2b95 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394f0d 2fbcc669-f80f-4096-97e2-242b817b2b95 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10180017735251369316,"high":2523931557934127279},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367190626617227,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:58.343823272+08:00","start-ts":457114699312922626,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114700571475971} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704346349574,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2fbcc669-f80f-4096-97e2-242b817b2b95 {"id":"2fbcc669-f80f-4096-97e2-242b817b2b95","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754215} /tidb/cdc/default/__cdc_meta__/capture/8dfb8894-2ecd-4813-b06d-0ac1bf49cdef {"id":"8dfb8894-2ecd-4813-b06d-0ac1bf49cdef","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754202} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394f0d 2fbcc669-f80f-4096-97e2-242b817b2b95 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394f0d 2fbcc669-f80f-4096-97e2-242b817b2b95 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10180017735251369316,"high":2523931557934127279},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367190626617227,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:58.343823272+08:00","start-ts":457114699312922626,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114700571475971} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704346349574,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2fbcc669-f80f-4096-97e2-242b817b2b95 {"id":"2fbcc669-f80f-4096-97e2-242b817b2b95","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754215} /tidb/cdc/default/__cdc_meta__/capture/8dfb8894-2ecd-4813-b06d-0ac1bf49cdef {"id":"8dfb8894-2ecd-4813-b06d-0ac1bf49cdef","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754202} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9394f0d 2fbcc669-f80f-4096-97e2-242b817b2b95 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394cc5 8dfb8894-2ecd-4813-b06d-0ac1bf49cdef /tidb/cdc/default/__cdc_meta__/owner/223195ffd9394f0d 2fbcc669-f80f-4096-97e2-242b817b2b95 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10180017735251369316,"high":2523931557934127279},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367190626617227,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:58.343823272+08:00","start-ts":457114699312922626,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114700571475971} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704346349574,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22ccd9fd-9024-412f-a4fb-3212e10d7593 {"id":"22ccd9fd-9024-412f-a4fb-3212e10d7593","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704084729864,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22ccd9fd-9024-412f-a4fb-3212e10d7593 {"id":"22ccd9fd-9024-412f-a4fb-3212e10d7593","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704084729864,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22ccd9fd-9024-412f-a4fb-3212e10d7593 {"id":"22ccd9fd-9024-412f-a4fb-3212e10d7593","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114704084729864,"progress":1}' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:18 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1401214014.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_L Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:19 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/657b392c-c236-4e68-b4ce-3b75291dff0c {"id":"657b392c-c236-4e68-b4ce-3b75291dff0c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/owner/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704276357141,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/657b392c-c236-4e68-b4ce-3b75291dff0c {"id":"657b392c-c236-4e68-b4ce-3b75291dff0c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/owner/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704276357141,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/657b392c-c236-4e68-b4ce-3b75291dff0c {"id":"657b392c-c236-4e68-b4ce-3b75291dff0c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/owner/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704276357141,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:10:19 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1333513337.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.15885.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.13126.out cli tso query --pd=http://127.0.0.1:2379 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully + set +x + tso='457114705916592130 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114705916592130 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:20 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1240112403.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.cli.13534.out cli tso query --pd=http://127.0.0.1:2379 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (11341) - No such process wait process 11341 exit for 1-th time... process 11341 already exit chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 + set +x + tso='457114706186862593 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114706186862593 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:21 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1592515927.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseK-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + tso='457114706325798913 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114706325798913 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:21 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1356513567.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114706335498241 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114706335498241 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:21 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1316213164.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01e49535-0f73-4a2b-a8b1-bc7c6f668813 {"id":"01e49535-0f73-4a2b-a8b1-bc7c6f668813","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754219} /tidb/cdc/default/__cdc_meta__/capture/22ccd9fd-9024-412f-a4fb-3212e10d7593 {"id":"22ccd9fd-9024-412f-a4fb-3212e10d7593","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f56e8 01e49535-0f73-4a2b-a8b1-bc7c6f668813 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f56e8 01e49535-0f73-4a2b-a8b1-bc7c6f668813 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704084729864,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01e49535-0f73-4a2b-a8b1-bc7c6f668813 {"id":"01e49535-0f73-4a2b-a8b1-bc7c6f668813","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754219} /tidb/cdc/default/__cdc_meta__/capture/22ccd9fd-9024-412f-a4fb-3212e10d7593 {"id":"22ccd9fd-9024-412f-a4fb-3212e10d7593","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f56e8 01e49535-0f73-4a2b-a8b1-bc7c6f668813 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f56e8 01e49535-0f73-4a2b-a8b1-bc7c6f668813 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704084729864,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01e49535-0f73-4a2b-a8b1-bc7c6f668813 {"id":"01e49535-0f73-4a2b-a8b1-bc7c6f668813","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754219} /tidb/cdc/default/__cdc_meta__/capture/22ccd9fd-9024-412f-a4fb-3212e10d7593 {"id":"22ccd9fd-9024-412f-a4fb-3212e10d7593","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f54ee 8ed8d6e4-a1c4-424c-bb1f-7da8fc7454e7 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd93f56e8 01e49535-0f73-4a2b-a8b1-bc7c6f668813 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f5682 22ccd9fd-9024-412f-a4fb-3212e10d7593 /tidb/cdc/default/__cdc_meta__/owner/223195ffd93f56e8 01e49535-0f73-4a2b-a8b1-bc7c6f668813 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5569667663057295696,"high":15486335412544235427},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367200255130049,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:00.304868896+08:00","start-ts":457114699837997057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701083181061} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704084729864,"progress":1}' + grep -q 'etcd info' + break + set +x chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_O Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:22 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/657b392c-c236-4e68-b4ce-3b75291dff0c {"id":"657b392c-c236-4e68-b4ce-3b75291dff0c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/capture/ea59e3a9-d73e-4503-ace7-71dba6c79dfc {"id":"ea59e3a9-d73e-4503-ace7-71dba6c79dfc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754219} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd970844f ea59e3a9-d73e-4503-ace7-71dba6c79dfc /tidb/cdc/default/__cdc_meta__/owner/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/owner/223195ffd970844f ea59e3a9-d73e-4503-ace7-71dba6c79dfc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704276357141,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/657b392c-c236-4e68-b4ce-3b75291dff0c {"id":"657b392c-c236-4e68-b4ce-3b75291dff0c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/capture/ea59e3a9-d73e-4503-ace7-71dba6c79dfc {"id":"ea59e3a9-d73e-4503-ace7-71dba6c79dfc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754219} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd970844f ea59e3a9-d73e-4503-ace7-71dba6c79dfc /tidb/cdc/default/__cdc_meta__/owner/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/owner/223195ffd970844f ea59e3a9-d73e-4503-ace7-71dba6c79dfc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704276357141,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/657b392c-c236-4e68-b4ce-3b75291dff0c {"id":"657b392c-c236-4e68-b4ce-3b75291dff0c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754216} /tidb/cdc/default/__cdc_meta__/capture/ea59e3a9-d73e-4503-ace7-71dba6c79dfc {"id":"ea59e3a9-d73e-4503-ace7-71dba6c79dfc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754219} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97082e2 4e24468e-5caa-4d85-9356-41c79eaabd84 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd970844f ea59e3a9-d73e-4503-ace7-71dba6c79dfc /tidb/cdc/default/__cdc_meta__/owner/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/owner/223195ffd970844f ea59e3a9-d73e-4503-ace7-71dba6c79dfc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114704276357141,"progress":1}' + grep -q 'etcd info' + break + set +x /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13085) - No such process wait process 13085 exit for 1-th time... process 13085 already exit [Fri Apr 4 16:10:20 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1327513277.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [2980 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6cf78b2-688f-48ef-bd4e-120331c91635 {"id":"c6cf78b2-688f-48ef-bd4e-120331c91635","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5065050923354093612,"high":7008916332082200506},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367166296562395,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:53.5448803+08:00","start-ts":457114698071408642,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699303485451} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114705988157453,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6cf78b2-688f-48ef-bd4e-120331c91635 {"id":"c6cf78b2-688f-48ef-bd4e-120331c91635","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5065050923354093612,"high":7008916332082200506},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367166296562395,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:53.5448803+08:00","start-ts":457114698071408642,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699303485451} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114705988157453,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6cf78b2-688f-48ef-bd4e-120331c91635 {"id":"c6cf78b2-688f-48ef-bd4e-120331c91635","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754197} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/__cdc_meta__/owner/223195ffd926cac4 c6cf78b2-688f-48ef-bd4e-120331c91635 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5065050923354093612,"high":7008916332082200506},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367166296562395,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:53.5448803+08:00","start-ts":457114698071408642,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699303485451} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114705988157453,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:23 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4facf821-d8e4-41ef-8e6e-cc6591707c0d {"id":"4facf821-d8e4-41ef-8e6e-cc6591707c0d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754220} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4facf821-d8e4-41ef-8e6e-cc6591707c0d {"id":"4facf821-d8e4-41ef-8e6e-cc6591707c0d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754220} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4facf821-d8e4-41ef-8e6e-cc6591707c0d {"id":"4facf821-d8e4-41ef-8e6e-cc6591707c0d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754220} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114705916592130 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.12466.out cli changefeed create --start-ts=457114705916592130 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367299086423050,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:23.47402767+08:00","start_ts":457114705916592130,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114705916592130,"checkpoint_ts":457114705916592130,"checkpoint_time":"2025-04-04 16:10:18.737","gid":{"low":10037357187106854639,"high":7309533088047575176}} PASS check diff failed 1-th time, retry later coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:24 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fce0a1c9-5573-4641-8640-1a4d2309a8c8 {"id":"fce0a1c9-5573-4641-8640-1a4d2309a8c8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754221} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fce0a1c9-5573-4641-8640-1a4d2309a8c8 {"id":"fce0a1c9-5573-4641-8640-1a4d2309a8c8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754221} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fce0a1c9-5573-4641-8640-1a4d2309a8c8 {"id":"fce0a1c9-5573-4641-8640-1a4d2309a8c8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754221} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114706186862593 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.15990.out cli changefeed create --start-ts=457114706186862593 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367301609395499,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:24.536280375+08:00","start_ts":457114706186862593,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114706186862593,"checkpoint_ts":457114706186862593,"checkpoint_time":"2025-04-04 16:10:19.768","gid":{"low":4558862218431368628,"high":4669594375052180648}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_M Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:24 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c8bae21a-e71d-4e12-afe6-f0cae31e57d6 {"id":"c8bae21a-e71d-4e12-afe6-f0cae31e57d6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754222} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c8bae21a-e71d-4e12-afe6-f0cae31e57d6 {"id":"c8bae21a-e71d-4e12-afe6-f0cae31e57d6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754222} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c8bae21a-e71d-4e12-afe6-f0cae31e57d6 {"id":"c8bae21a-e71d-4e12-afe6-f0cae31e57d6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754222} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114706325798913 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.cli.13635.out cli changefeed create --start-ts=457114706325798913 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367304856166827,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:25.057376155+08:00","start_ts":457114706325798913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114706325798913,"checkpoint_ts":457114706325798913,"checkpoint_time":"2025-04-04 16:10:20.298","gid":{"low":3769471121336011646,"high":13664478006238333864}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:24 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8323e3e8-022d-48d0-819d-7fcee2e9e874 {"id":"8323e3e8-022d-48d0-819d-7fcee2e9e874","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754222} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a800e4 8323e3e8-022d-48d0-819d-7fcee2e9e874 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a800e4 8323e3e8-022d-48d0-819d-7fcee2e9e874' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8323e3e8-022d-48d0-819d-7fcee2e9e874 {"id":"8323e3e8-022d-48d0-819d-7fcee2e9e874","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754222} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a800e4 8323e3e8-022d-48d0-819d-7fcee2e9e874 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a800e4 8323e3e8-022d-48d0-819d-7fcee2e9e874' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8323e3e8-022d-48d0-819d-7fcee2e9e874 {"id":"8323e3e8-022d-48d0-819d-7fcee2e9e874","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754222} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a800e4 8323e3e8-022d-48d0-819d-7fcee2e9e874 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a800e4 8323e3e8-022d-48d0-819d-7fcee2e9e874' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114706335498241 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.13224.out cli changefeed create --start-ts=457114706335498241 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367305182583740,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:25.100357698+08:00","start_ts":457114706335498241,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114706335498241,"checkpoint_ts":457114706335498241,"checkpoint_time":"2025-04-04 16:10:20.335","gid":{"low":6935560052145240848,"high":2121570029465933459}} PASS Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13264) - No such process wait process 13264 exit for 1-th time... process 13264 already exit coverage: 2.7% of statements in github.com/pingcap/ticdc/... command: check_safepoint_forward http://127.0.0.1:2379 7489367088931747056 457114699174772735 check diff failed 2-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_F Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 3-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseC-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13340) - No such process wait process 13340 exit for 1-th time... process 13340 already exit [Fri Apr 4 16:10:25 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1344513447.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later safepoint 457114699174772735 is not forward run task failed 5-th time, retry later + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (12406) - No such process wait process 12406 exit for 1-th time... process 12406 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:10:27 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1253712539.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 table fail_over_ddl_test.test2 exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseG-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table fail_over_ddl_test.test2 exists table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15930) - No such process wait process 15930 exit for 1-th time... process 15930 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:10:28 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1605916061.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_K Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13570) - No such process wait process 13570 exit for 1-th time... process 13570 already exit [Fri Apr 4 16:10:29 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1369613698.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89b431e0-6d0a-49e0-8a10-7735868c9556 {"id":"89b431e0-6d0a-49e0-8a10-7735868c9556","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754226} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707251691535,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89b431e0-6d0a-49e0-8a10-7735868c9556 {"id":"89b431e0-6d0a-49e0-8a10-7735868c9556","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754226} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707251691535,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89b431e0-6d0a-49e0-8a10-7735868c9556 {"id":"89b431e0-6d0a-49e0-8a10-7735868c9556","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754226} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707251691535,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":1}' + echo '{"count":1}' {"count":1} ++ echo '{"count":1}' ++ jq -r .count + value=1 + '[' -1 == -1 ']' + '[' 1 '!=' null ']' + exit 0 [Fri Apr 4 16:10:28 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1352213524.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13167) - No such process wait process 13167 exit for 1-th time... process 13167 already exit cdc.test: no process found process cdc.test already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (9731) - No such process wait process 9731 exit for 1-th time... process 9731 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367355570179202 TiDB Cluster ID. tikv_gc_leader_uuid 657ff67c940002e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:11929, start at 2025-04-04 16:10:28.347580198 +0800 CST m=+3.352676431 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:28.357 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:28.354 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:28.354 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367355570179202 TiDB Cluster ID. tikv_gc_leader_uuid 657ff67c940002e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:11929, start at 2025-04-04 16:10:28.347580198 +0800 CST m=+3.352676431 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:28.357 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:28.354 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:28.354 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367351482183599 TiDB Cluster ID. tikv_gc_leader_uuid 657ff67c9f80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:11994, start at 2025-04-04 16:10:28.360749618 +0800 CST m=+3.210224524 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:28.367 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:28.350 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:28.350 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCase-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:30 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 {"id":"246efdda-fb43-4fd4-bd0b-73b0c50bf9d5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754227} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10037357187106854639,"high":7309533088047575176},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367299086423050,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:23.47402767+08:00","start-ts":457114705916592130,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707148668939} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707528777762,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 {"id":"246efdda-fb43-4fd4-bd0b-73b0c50bf9d5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754227} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10037357187106854639,"high":7309533088047575176},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367299086423050,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:23.47402767+08:00","start-ts":457114705916592130,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707148668939} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707528777762,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 {"id":"246efdda-fb43-4fd4-bd0b-73b0c50bf9d5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754227} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10037357187106854639,"high":7309533088047575176},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367299086423050,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:23.47402767+08:00","start-ts":457114705916592130,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707148668939} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707528777762,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:30 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1261612618.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.13210.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367362330872194 TiDB Cluster ID. tikv_gc_leader_uuid 657ff67ec400011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:11822, start at 2025-04-04 16:10:30.573058393 +0800 CST m=+3.501479208 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:30.646 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:30.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:30.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367362330872194 TiDB Cluster ID. tikv_gc_leader_uuid 657ff67ec400011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:11822, start at 2025-04-04 16:10:30.573058393 +0800 CST m=+3.501479208 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:30.646 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:30.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:30.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367360608636158 TiDB Cluster ID. tikv_gc_leader_uuid 657ff67ef600007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:11894, start at 2025-04-04 16:10:30.755661169 +0800 CST m=+3.575044501 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:30.764 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:30.744 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:30.744 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f38de8-025f-4849-8376-a984760d8e6c {"id":"16f38de8-025f-4849-8376-a984760d8e6c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754228} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4558862218431368628,"high":4669594375052180648},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367301609395499,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:24.536280375+08:00","start-ts":457114706186862593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707432046598} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707799048193,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f38de8-025f-4849-8376-a984760d8e6c {"id":"16f38de8-025f-4849-8376-a984760d8e6c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754228} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4558862218431368628,"high":4669594375052180648},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367301609395499,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:24.536280375+08:00","start-ts":457114706186862593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707432046598} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707799048193,"progress":1}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f38de8-025f-4849-8376-a984760d8e6c {"id":"16f38de8-025f-4849-8376-a984760d8e6c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754228} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4558862218431368628,"high":4669594375052180648},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367301609395499,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:24.536280375+08:00","start-ts":457114706186862593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707432046598} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707799048193,"progress":1}' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:31 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1612916131.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fdae1a7-65b0-45e1-8331-ae2a90acfddf {"id":"9fdae1a7-65b0-45e1-8331-ae2a90acfddf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754229} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707951091729,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fdae1a7-65b0-45e1-8331-ae2a90acfddf {"id":"9fdae1a7-65b0-45e1-8331-ae2a90acfddf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754229} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707951091729,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fdae1a7-65b0-45e1-8331-ae2a90acfddf {"id":"9fdae1a7-65b0-45e1-8331-ae2a90acfddf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754229} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707951091729,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:10:32 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1376713769.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89b431e0-6d0a-49e0-8a10-7735868c9556 {"id":"89b431e0-6d0a-49e0-8a10-7735868c9556","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754226} /tidb/cdc/default/__cdc_meta__/capture/d2659204-9e3b-4eaa-ad44-221454f13c9a {"id":"d2659204-9e3b-4eaa-ad44-221454f13c9a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754229} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97085ed d2659204-9e3b-4eaa-ad44-221454f13c9a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/__cdc_meta__/owner/223195ffd97085ed d2659204-9e3b-4eaa-ad44-221454f13c9a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114708313374721,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89b431e0-6d0a-49e0-8a10-7735868c9556 {"id":"89b431e0-6d0a-49e0-8a10-7735868c9556","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754226} /tidb/cdc/default/__cdc_meta__/capture/d2659204-9e3b-4eaa-ad44-221454f13c9a {"id":"d2659204-9e3b-4eaa-ad44-221454f13c9a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754229} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97085ed d2659204-9e3b-4eaa-ad44-221454f13c9a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/__cdc_meta__/owner/223195ffd97085ed d2659204-9e3b-4eaa-ad44-221454f13c9a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114708313374721,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89b431e0-6d0a-49e0-8a10-7735868c9556 {"id":"89b431e0-6d0a-49e0-8a10-7735868c9556","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754226} /tidb/cdc/default/__cdc_meta__/capture/d2659204-9e3b-4eaa-ad44-221454f13c9a {"id":"d2659204-9e3b-4eaa-ad44-221454f13c9a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754229} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97083eb 657b392c-c236-4e68-b4ce-3b75291dff0c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd97085ed d2659204-9e3b-4eaa-ad44-221454f13c9a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9708587 89b431e0-6d0a-49e0-8a10-7735868c9556 /tidb/cdc/default/__cdc_meta__/owner/223195ffd97085ed d2659204-9e3b-4eaa-ad44-221454f13c9a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2903904039547186801,"high":14969412283366187181},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367243016454897,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:10.52520532+08:00","start-ts":457114702506885121,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114703765176321} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114708313374721,"progress":1}' + grep -q 'etcd info' + break + set +x chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:32 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1004010042.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='457114709381087233 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114709381087233 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:33 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1324613248.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.13114.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:33 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1edf495e-fc6d-4190-8aa0-26efda5edc6d {"id":"1edf495e-fc6d-4190-8aa0-26efda5edc6d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754231} /tidb/cdc/default/__cdc_meta__/capture/246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 {"id":"246efdda-fb43-4fd4-bd0b-73b0c50bf9d5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754227} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10037357187106854639,"high":7309533088047575176},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367299086423050,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:23.47402767+08:00","start-ts":457114705916592130,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707148668939} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707528777762,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1edf495e-fc6d-4190-8aa0-26efda5edc6d {"id":"1edf495e-fc6d-4190-8aa0-26efda5edc6d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754231} /tidb/cdc/default/__cdc_meta__/capture/246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 {"id":"246efdda-fb43-4fd4-bd0b-73b0c50bf9d5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754227} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10037357187106854639,"high":7309533088047575176},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367299086423050,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:23.47402767+08:00","start-ts":457114705916592130,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707148668939} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707528777762,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1edf495e-fc6d-4190-8aa0-26efda5edc6d {"id":"1edf495e-fc6d-4190-8aa0-26efda5edc6d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754231} /tidb/cdc/default/__cdc_meta__/capture/246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 {"id":"246efdda-fb43-4fd4-bd0b-73b0c50bf9d5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754227} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be3e6 4facf821-d8e4-41ef-8e6e-cc6591707c0d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be4c3 246efdda-fb43-4fd4-bd0b-73b0c50bf9d5 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10037357187106854639,"high":7309533088047575176},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367299086423050,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:23.47402767+08:00","start-ts":457114705916592130,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707148668939} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707528777762,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.12688.out cli capture list chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_C Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367375967016301 TiDB Cluster ID. tikv_gc_leader_uuid 657ff680da8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:26061, start at 2025-04-04 16:10:32.705705061 +0800 CST m=+3.082677611 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:32.714 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:32.718 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:32.718 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367375967016301 TiDB Cluster ID. tikv_gc_leader_uuid 657ff680da8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:26061, start at 2025-04-04 16:10:32.705705061 +0800 CST m=+3.082677611 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:32.714 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:32.718 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:32.718 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367376135218241 TiDB Cluster ID. tikv_gc_leader_uuid 657ff681708000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:26123, start at 2025-04-04 16:10:33.297349942 +0800 CST m=+3.500067981 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:33.304 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:33.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:33.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367375875505646 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6814b00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:14260, start at 2025-04-04 16:10:33.13665223 +0800 CST m=+3.456198688 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:33.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:33.140 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:33.140 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367375875505646 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6814b00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:14260, start at 2025-04-04 16:10:33.13665223 +0800 CST m=+3.456198688 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:33.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:33.140 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:33.140 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367373196955779 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6817d8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:14341, start at 2025-04-04 16:10:33.355090507 +0800 CST m=+3.508638172 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:33.363 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:33.334 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:33.334 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:34 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f38de8-025f-4849-8376-a984760d8e6c {"id":"16f38de8-025f-4849-8376-a984760d8e6c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754228} /tidb/cdc/default/__cdc_meta__/capture/1baf3297-5698-4bfc-8a2f-5d29437346fc {"id":"1baf3297-5698-4bfc-8a2f-5d29437346fc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4558862218431368628,"high":4669594375052180648},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367301609395499,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:24.536280375+08:00","start-ts":457114706186862593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707432046598} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707799048193,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f38de8-025f-4849-8376-a984760d8e6c {"id":"16f38de8-025f-4849-8376-a984760d8e6c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754228} /tidb/cdc/default/__cdc_meta__/capture/1baf3297-5698-4bfc-8a2f-5d29437346fc {"id":"1baf3297-5698-4bfc-8a2f-5d29437346fc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4558862218431368628,"high":4669594375052180648},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367301609395499,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:24.536280375+08:00","start-ts":457114706186862593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707432046598} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707799048193,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f38de8-025f-4849-8376-a984760d8e6c {"id":"16f38de8-025f-4849-8376-a984760d8e6c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754228} /tidb/cdc/default/__cdc_meta__/capture/1baf3297-5698-4bfc-8a2f-5d29437346fc {"id":"1baf3297-5698-4bfc-8a2f-5d29437346fc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbae6 fce0a1c9-5573-4641-8640-1a4d2309a8c8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbbc4 16f38de8-025f-4849-8376-a984760d8e6c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4558862218431368628,"high":4669594375052180648},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367301609395499,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:24.536280375+08:00","start-ts":457114706186862593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707432046598} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707799048193,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.16190.out cli capture list + set +x + ans='[ { "id": "246efdda-fb43-4fd4-bd0b-73b0c50bf9d5", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "1edf495e-fc6d-4190-8aa0-26efda5edc6d", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"246efdda-fb43-4fd4-bd0b-73b0c50bf9d5",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"1edf495e-fc6d-4190-8aa0-26efda5edc6d",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=1edf495e-fc6d-4190-8aa0-26efda5edc6d + '[' -z 1edf495e-fc6d-4190-8aa0-26efda5edc6d ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 1edf495e-fc6d-4190-8aa0-26efda5edc6d + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.12718.out cli changefeed move-table -c test -t 114 -d 1edf495e-fc6d-4190-8aa0-26efda5edc6d + set +x + tso='457114709914288129 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114709914288129 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:35 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1314813152.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/877aabb4-13d7-4763-ad0b-025f51cdaa5b {"id":"877aabb4-13d7-4763-ad0b-025f51cdaa5b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/capture/9fdae1a7-65b0-45e1-8331-ae2a90acfddf {"id":"9fdae1a7-65b0-45e1-8331-ae2a90acfddf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754229} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fda28 877aabb4-13d7-4763-ad0b-025f51cdaa5b /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fda28 877aabb4-13d7-4763-ad0b-025f51cdaa5b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707951091729,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/877aabb4-13d7-4763-ad0b-025f51cdaa5b {"id":"877aabb4-13d7-4763-ad0b-025f51cdaa5b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/capture/9fdae1a7-65b0-45e1-8331-ae2a90acfddf {"id":"9fdae1a7-65b0-45e1-8331-ae2a90acfddf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754229} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fda28 877aabb4-13d7-4763-ad0b-025f51cdaa5b /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fda28 877aabb4-13d7-4763-ad0b-025f51cdaa5b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707951091729,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/877aabb4-13d7-4763-ad0b-025f51cdaa5b {"id":"877aabb4-13d7-4763-ad0b-025f51cdaa5b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/capture/9fdae1a7-65b0-45e1-8331-ae2a90acfddf {"id":"9fdae1a7-65b0-45e1-8331-ae2a90acfddf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754229} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd8e6 c8bae21a-e71d-4e12-afe6-f0cae31e57d6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fda28 877aabb4-13d7-4763-ad0b-025f51cdaa5b /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fda28 877aabb4-13d7-4763-ad0b-025f51cdaa5b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707951091729,"progress":1}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9198ac62-a4f5-4477-85c4-687acc630a62 {"id":"9198ac62-a4f5-4477-85c4-687acc630a62","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/capture/e5f189eb-6f3b-4e10-b8a2-088ffb019a75 {"id":"e5f189eb-6f3b-4e10-b8a2-088ffb019a75","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754203} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930bc27 9198ac62-a4f5-4477-85c4-687acc630a62 /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/__cdc_meta__/owner/223195ffd930bc27 9198ac62-a4f5-4477-85c4-687acc630a62 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9750020843988387933,"high":8751326180401707658},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367178140310957,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:56.110054484+08:00","start-ts":457114698736205825,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699981389829} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114703769370635,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9198ac62-a4f5-4477-85c4-687acc630a62 {"id":"9198ac62-a4f5-4477-85c4-687acc630a62","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/capture/e5f189eb-6f3b-4e10-b8a2-088ffb019a75 {"id":"e5f189eb-6f3b-4e10-b8a2-088ffb019a75","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754203} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930bc27 9198ac62-a4f5-4477-85c4-687acc630a62 /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/__cdc_meta__/owner/223195ffd930bc27 9198ac62-a4f5-4477-85c4-687acc630a62 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9750020843988387933,"high":8751326180401707658},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367178140310957,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:56.110054484+08:00","start-ts":457114698736205825,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699981389829} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114703769370635,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9198ac62-a4f5-4477-85c4-687acc630a62 {"id":"9198ac62-a4f5-4477-85c4-687acc630a62","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/capture/e5f189eb-6f3b-4e10-b8a2-088ffb019a75 {"id":"e5f189eb-6f3b-4e10-b8a2-088ffb019a75","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754203} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd930bc27 9198ac62-a4f5-4477-85c4-687acc630a62 /tidb/cdc/default/__cdc_meta__/owner/223195ffd930b727 e5f189eb-6f3b-4e10-b8a2-088ffb019a75 /tidb/cdc/default/__cdc_meta__/owner/223195ffd930bc27 9198ac62-a4f5-4477-85c4-687acc630a62 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9750020843988387933,"high":8751326180401707658},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367178140310957,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:09:56.110054484+08:00","start-ts":457114698736205825,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114699981389829} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114703769370635,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.27461.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 2-th time, retry later command: check_safepoint_forward http://127.0.0.1:2379 7489367088931747056 457114699174772735 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.15674.out cli tso query --pd=http://127.0.0.1:2379 + set +x + ans='[ { "id": "16f38de8-025f-4849-8376-a984760d8e6c", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "1baf3297-5698-4bfc-8a2f-5d29437346fc", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"16f38de8-025f-4849-8376-a984760d8e6c",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"1baf3297-5698-4bfc-8a2f-5d29437346fc",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=1baf3297-5698-4bfc-8a2f-5d29437346fc + '[' -z 1baf3297-5698-4bfc-8a2f-5d29437346fc ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 1baf3297-5698-4bfc-8a2f-5d29437346fc + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.16220.out cli changefeed move-table -c test -t 114 -d 1baf3297-5698-4bfc-8a2f-5d29437346fc chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_G Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:36 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a731df69-5623-4d7f-9881-00aa3490edb8 {"id":"a731df69-5623-4d7f-9881-00aa3490edb8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754233} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a731df69-5623-4d7f-9881-00aa3490edb8 {"id":"a731df69-5623-4d7f-9881-00aa3490edb8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754233} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a731df69-5623-4d7f-9881-00aa3490edb8 {"id":"a731df69-5623-4d7f-9881-00aa3490edb8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754233} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114709381087233 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.13313.out cli changefeed create --start-ts=457114709381087233 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367355570179202,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:36.672698071+08:00","start_ts":457114709381087233,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114709381087233,"checkpoint_ts":457114709381087233,"checkpoint_time":"2025-04-04 16:10:31.953","gid":{"low":237186811318878080,"high":15219283008584226746}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... safepoint 457114699174772735 is not forward run task failed 6-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later + set +x Command succeeded on attempt 1 + set +x + tso='457114710590881793 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114710590881793 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:37 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.2749527497.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 + set +x + tso='457114710610018305 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114710610018305 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:38 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1571515717.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 not exists for 1-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_J Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseA-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Upstream TiDB... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:38 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7057dfe-85fb-44f3-aae3-a11ca8b973f0 {"id":"b7057dfe-85fb-44f3-aae3-a11ca8b973f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754235} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7057dfe-85fb-44f3-aae3-a11ca8b973f0 {"id":"b7057dfe-85fb-44f3-aae3-a11ca8b973f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754235} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7057dfe-85fb-44f3-aae3-a11ca8b973f0 {"id":"b7057dfe-85fb-44f3-aae3-a11ca8b973f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754235} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114709914288129 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.13211.out cli changefeed create --start-ts=457114709914288129 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367362330872194,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:38.758872607+08:00","start_ts":457114709914288129,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114709914288129,"checkpoint_ts":457114709914288129,"checkpoint_time":"2025-04-04 16:10:33.987","gid":{"low":4922064178680103469,"high":15056557442061542273}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367390112237633 TiDB Cluster ID. tikv_gc_leader_uuid 657ff686a840012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:12437, start at 2025-04-04 16:10:38.6465439 +0800 CST m=+3.407016886 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:38.654 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:38.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:38.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367390112237633 TiDB Cluster ID. tikv_gc_leader_uuid 657ff686a840012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:12437, start at 2025-04-04 16:10:38.6465439 +0800 CST m=+3.407016886 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:38.654 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:38.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:38.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367399566030310 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6868f40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:12508, start at 2025-04-04 16:10:38.535733583 +0800 CST m=+3.183588362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:38.543 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:38.525 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:38.525 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/error.log [2025/04/04 16:10:39.556 +08:00] [DEBUG] [pd.rs:582] ["update latency statistics for raftstore-v1"] [tick=424] [thread_id=88] [2025/04/04 16:10:39.556 +08:00] [DEBUG] [pool.rs:146] ["scheduling task inspect raftstore latency: RaftDisk"] [thread_id=88] [2025/04/04 16:10:39.556 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="inspect raftstore latency: RaftDisk"] [thread_id=30] [2025/04/04 16:10:39.556 +08:00] [DEBUG] [pool.rs:146] ["scheduling task compute slow score: id 425, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(18.675ยตs), store_process_duration: Some(6.647ยตs), store_write_duration: Some(7.01ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=80] [2025/04/04 16:10:39.556 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="compute slow score: id 425, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(18.675ยตs), store_process_duration: Some(6.647ยตs), store_write_duration: Some(7.01ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=30] + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table fail_over_ddl_test.test2 exists + set +x Command succeeded on attempt 1 command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13251) - No such process wait process 13251 exit for 1-th time... process 13251 already exit [Fri Apr 4 16:10:40 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1338213384.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (12542) - No such process wait process 12542 exit for 1-th time... process 12542 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Fri Apr 4 16:10:39 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1331413316.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:40 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e401c8b-6a15-4796-85b3-01160932b36f {"id":"4e401c8b-6a15-4796-85b3-01160932b36f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754238} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e401c8b-6a15-4796-85b3-01160932b36f {"id":"4e401c8b-6a15-4796-85b3-01160932b36f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754238} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e401c8b-6a15-4796-85b3-01160932b36f {"id":"4e401c8b-6a15-4796-85b3-01160932b36f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754238} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114710590881793 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.27552.out cli changefeed create --start-ts=457114710590881793 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367375967016301,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:41.349883251+08:00","start_ts":457114710590881793,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114710590881793,"checkpoint_ts":457114710590881793,"checkpoint_time":"2025-04-04 16:10:36.568","gid":{"low":2544482256124478383,"high":12326019957456358587}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:41 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c715cf08-ed13-42c6-a971-254e75f1136c {"id":"c715cf08-ed13-42c6-a971-254e75f1136c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754238} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c715cf08-ed13-42c6-a971-254e75f1136c {"id":"c715cf08-ed13-42c6-a971-254e75f1136c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754238} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c715cf08-ed13-42c6-a971-254e75f1136c {"id":"c715cf08-ed13-42c6-a971-254e75f1136c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754238} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114710610018305 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.15778.out cli changefeed create --start-ts=457114710610018305 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367375875505646,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:41.417314802+08:00","start_ts":457114710610018305,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114710610018305,"checkpoint_ts":457114710610018305,"checkpoint_time":"2025-04-04 16:10:36.641","gid":{"low":13206254903965006868,"high":5483312206236578955}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' TEST FAILED: OUTPUT CONTAINS 'fail_over_ddl_test' ____________________________________ *************************** 1. row *************************** Database: INFORMATION_SCHEMA *************************** 2. row *************************** Database: METRICS_SCHEMA *************************** 3. row *************************** Database: PERFORMANCE_SCHEMA *************************** 4. row *************************** Database: fail_over_ddl_test *************************** 5. row *************************** Database: mysql *************************** 6. row *************************** Database: sys *************************** 7. row *************************** Database: test *************************** 8. row *************************** Database: tidb_cdc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.cli.13707.out cli tso query --pd=http://127.0.0.1:2379 table fail_over_ddl_test.test2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:42 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [2982 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/95bbfba8-4fdb-4943-ab98-579acfb4f206 {"id":"95bbfba8-4fdb-4943-ab98-579acfb4f206","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754239} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6935560052145240848,"high":2121570029465933459},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367305182583740,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.100357698+08:00","start-ts":457114706335498241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707580682246} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707947683875,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/95bbfba8-4fdb-4943-ab98-579acfb4f206 {"id":"95bbfba8-4fdb-4943-ab98-579acfb4f206","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754239} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6935560052145240848,"high":2121570029465933459},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367305182583740,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.100357698+08:00","start-ts":457114706335498241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707580682246} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707947683875,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/95bbfba8-4fdb-4943-ab98-579acfb4f206 {"id":"95bbfba8-4fdb-4943-ab98-579acfb4f206","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754239} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6935560052145240848,"high":2121570029465933459},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367305182583740,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.100357698+08:00","start-ts":457114706335498241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707580682246} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707947683875,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:42 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1338913391.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + set +x Command succeeded on attempt 1 + set +x Command succeeded on attempt 1 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table fail_over_ddl_test.test2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367415843588156 TiDB Cluster ID. tikv_gc_leader_uuid 657ff689fd00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:17183, start at 2025-04-04 16:10:42.043614131 +0800 CST m=+3.263263072 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:42.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:42.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:42.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367415843588156 TiDB Cluster ID. tikv_gc_leader_uuid 657ff689fd00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:17183, start at 2025-04-04 16:10:42.043614131 +0800 CST m=+3.263263072 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:42.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:42.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:42.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367411900315623 TiDB Cluster ID. tikv_gc_leader_uuid 657ff68a1940005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-63s8p--0rlz4, pid:17258, start at 2025-04-04 16:10:42.154829702 +0800 CST m=+3.207034874 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:42.161 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:42.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:42.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/error.log Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_C/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='457114712047353857 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114712047353857 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:43 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1373813740.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51b854d5-378c-4cad-9dad-3e7ad4a4202e {"id":"51b854d5-378c-4cad-9dad-3e7ad4a4202e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754241} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":237186811318878080,"high":15219283008584226746},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367355570179202,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:36.672698071+08:00","start-ts":457114709381087233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114710613426182} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114710980427792,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51b854d5-378c-4cad-9dad-3e7ad4a4202e {"id":"51b854d5-378c-4cad-9dad-3e7ad4a4202e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754241} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":237186811318878080,"high":15219283008584226746},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367355570179202,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:36.672698071+08:00","start-ts":457114709381087233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114710613426182} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114710980427792,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51b854d5-378c-4cad-9dad-3e7ad4a4202e {"id":"51b854d5-378c-4cad-9dad-3e7ad4a4202e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754241} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":237186811318878080,"high":15219283008584226746},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367355570179202,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:36.672698071+08:00","start-ts":457114709381087233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114710613426182} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114710980427792,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:43 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1345613458.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13155) - No such process wait process 13155 exit for 1-th time... process 13155 already exit [Fri Apr 4 16:10:44 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1328613288.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table fail_over_ddl_test.test2 exists + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.18490.out cli tso query --pd=http://127.0.0.1:2379 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15720) - No such process wait process 15720 exit for 1-th time... process 15720 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:10:45 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1585315855.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:45 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78b69aa3-0140-46b5-bc0f-6c42ce922694 {"id":"78b69aa3-0140-46b5-bc0f-6c42ce922694","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754242} /tidb/cdc/default/__cdc_meta__/capture/95bbfba8-4fdb-4943-ab98-579acfb4f206 {"id":"95bbfba8-4fdb-4943-ab98-579acfb4f206","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754239} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6935560052145240848,"high":2121570029465933459},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367305182583740,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.100357698+08:00","start-ts":457114706335498241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707580682246} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707947683875,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78b69aa3-0140-46b5-bc0f-6c42ce922694 {"id":"78b69aa3-0140-46b5-bc0f-6c42ce922694","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754242} /tidb/cdc/default/__cdc_meta__/capture/95bbfba8-4fdb-4943-ab98-579acfb4f206 {"id":"95bbfba8-4fdb-4943-ab98-579acfb4f206","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754239} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6935560052145240848,"high":2121570029465933459},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367305182583740,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.100357698+08:00","start-ts":457114706335498241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707580682246} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707947683875,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78b69aa3-0140-46b5-bc0f-6c42ce922694 {"id":"78b69aa3-0140-46b5-bc0f-6c42ce922694","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754242} /tidb/cdc/default/__cdc_meta__/capture/95bbfba8-4fdb-4943-ab98-579acfb4f206 {"id":"95bbfba8-4fdb-4943-ab98-579acfb4f206","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754239} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a801fa 95bbfba8-4fdb-4943-ab98-579acfb4f206 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6935560052145240848,"high":2121570029465933459},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367305182583740,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.100357698+08:00","start-ts":457114706335498241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707580682246} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114707947683875,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.13462.out cli capture list table fail_over_ddl_test.test2 exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367424752522402 TiDB Cluster ID. tikv_gc_leader_uuid 657ff68d9740004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:15005, start at 2025-04-04 16:10:45.730238346 +0800 CST m=+3.587344724 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:45.738 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:45.745 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:45.745 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367424752522402 TiDB Cluster ID. tikv_gc_leader_uuid 657ff68d9740004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:15005, start at 2025-04-04 16:10:45.730238346 +0800 CST m=+3.587344724 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:45.738 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:45.745 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:45.745 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367430691953595 TiDB Cluster ID. tikv_gc_leader_uuid 657ff68d9080007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-3mwl6--8t52h, pid:15059, start at 2025-04-04 16:10:45.736582662 +0800 CST m=+3.466032741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:45.744 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:45.748 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:45.748 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_G/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:46 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f5cd251-7118-4a06-abe8-830d59e3fd7b {"id":"7f5cd251-7118-4a06-abe8-830d59e3fd7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754243} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f5cd251-7118-4a06-abe8-830d59e3fd7b {"id":"7f5cd251-7118-4a06-abe8-830d59e3fd7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754243} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f5cd251-7118-4a06-abe8-830d59e3fd7b {"id":"7f5cd251-7118-4a06-abe8-830d59e3fd7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754243} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114712047353857 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.cli.13796.out cli changefeed create --start-ts=457114712047353857 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367390112237633,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:46.902925948+08:00","start_ts":457114712047353857,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114712047353857,"checkpoint_ts":457114712047353857,"checkpoint_time":"2025-04-04 16:10:42.124","gid":{"low":955649239624675763,"high":18012802736310228379}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (27500) - No such process wait process 27500 exit for 1-th time... process 27500 already exit [Fri Apr 4 16:10:46 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.2763027632.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":1}' + echo '{"count":1}' {"count":1} ++ echo '{"count":1}' ++ jq -r .count + value=1 + '[' -1 == -1 ']' + '[' 1 '!=' null ']' + exit 0 [Fri Apr 4 16:10:46 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1280012802.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + ans='[ { "id": "95bbfba8-4fdb-4943-ab98-579acfb4f206", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "78b69aa3-0140-46b5-bc0f-6c42ce922694", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"95bbfba8-4fdb-4943-ab98-579acfb4f206",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"78b69aa3-0140-46b5-bc0f-6c42ce922694",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=78b69aa3-0140-46b5-bc0f-6c42ce922694 + '[' -z 78b69aa3-0140-46b5-bc0f-6c42ce922694 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 78b69aa3-0140-46b5-bc0f-6c42ce922694 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.13491.out cli changefeed move-table -c test -t 114 -d 78b69aa3-0140-46b5-bc0f-6c42ce922694 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367431094912519 TiDB Cluster ID. tikv_gc_leader_uuid 657ff68e4880004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:14352, start at 2025-04-04 16:10:46.439110528 +0800 CST m=+3.544672887 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:46.448 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:46.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:46.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367431094912519 TiDB Cluster ID. tikv_gc_leader_uuid 657ff68e4880004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:14352, start at 2025-04-04 16:10:46.439110528 +0800 CST m=+3.544672887 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:46.448 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:46.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:46.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367428866633719 TiDB Cluster ID. tikv_gc_leader_uuid 657ff68e478000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:14424, start at 2025-04-04 16:10:46.460865359 +0800 CST m=+3.430264330 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:46.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:46.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:46.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51b854d5-378c-4cad-9dad-3e7ad4a4202e {"id":"51b854d5-378c-4cad-9dad-3e7ad4a4202e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754241} /tidb/cdc/default/__cdc_meta__/capture/f228b299-e42c-40f0-a67a-08786bb9b1a7 {"id":"f228b299-e42c-40f0-a67a-08786bb9b1a7","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd6028 f228b299-e42c-40f0-a67a-08786bb9b1a7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd6028 f228b299-e42c-40f0-a67a-08786bb9b1a7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":237186811318878080,"high":15219283008584226746},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367355570179202,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:36.672698071+08:00","start-ts":457114709381087233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114710613426182} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114710993272869,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51b854d5-378c-4cad-9dad-3e7ad4a4202e {"id":"51b854d5-378c-4cad-9dad-3e7ad4a4202e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754241} /tidb/cdc/default/__cdc_meta__/capture/f228b299-e42c-40f0-a67a-08786bb9b1a7 {"id":"f228b299-e42c-40f0-a67a-08786bb9b1a7","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd6028 f228b299-e42c-40f0-a67a-08786bb9b1a7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd6028 f228b299-e42c-40f0-a67a-08786bb9b1a7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":237186811318878080,"high":15219283008584226746},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367355570179202,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:36.672698071+08:00","start-ts":457114709381087233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114710613426182} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114710993272869,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51b854d5-378c-4cad-9dad-3e7ad4a4202e {"id":"51b854d5-378c-4cad-9dad-3e7ad4a4202e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754241} /tidb/cdc/default/__cdc_meta__/capture/f228b299-e42c-40f0-a67a-08786bb9b1a7 {"id":"f228b299-e42c-40f0-a67a-08786bb9b1a7","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5ee6 a731df69-5623-4d7f-9881-00aa3490edb8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd6028 f228b299-e42c-40f0-a67a-08786bb9b1a7 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd6028 f228b299-e42c-40f0-a67a-08786bb9b1a7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":237186811318878080,"high":15219283008584226746},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367355570179202,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:36.672698071+08:00","start-ts":457114709381087233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114710613426182} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114710993272869,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.13528.out cli capture list + set +x + tso='457114712998150145 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114712998150145 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:47 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1851818520.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5999f63-d209-462c-b50e-03c2d4b94130 {"id":"f5999f63-d209-462c-b50e-03c2d4b94130","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4922064178680103469,"high":15056557442061542273},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367362330872194,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:38.758872607+08:00","start-ts":457114709914288129,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711159472135} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114711670652950,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5999f63-d209-462c-b50e-03c2d4b94130 {"id":"f5999f63-d209-462c-b50e-03c2d4b94130","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4922064178680103469,"high":15056557442061542273},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367362330872194,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:38.758872607+08:00","start-ts":457114709914288129,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711159472135} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114711670652950,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5999f63-d209-462c-b50e-03c2d4b94130 {"id":"f5999f63-d209-462c-b50e-03c2d4b94130","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4922064178680103469,"high":15056557442061542273},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367362330872194,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:38.758872607+08:00","start-ts":457114709914288129,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711159472135} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114711670652950,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:47 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1336713369.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_A Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... command: run_sql 'show databases;' 127.0.0.1 3306 && check_not_contains 'fail_over_ddl_test' run task successfully check diff failed 1-th time, retry later + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + set +x + ans='[ { "id": "51b854d5-378c-4cad-9dad-3e7ad4a4202e", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "f228b299-e42c-40f0-a67a-08786bb9b1a7", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"51b854d5-378c-4cad-9dad-3e7ad4a4202e",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"f228b299-e42c-40f0-a67a-08786bb9b1a7",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=f228b299-e42c-40f0-a67a-08786bb9b1a7 + '[' -z f228b299-e42c-40f0-a67a-08786bb9b1a7 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d f228b299-e42c-40f0-a67a-08786bb9b1a7 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.13562.out cli changefeed move-table -c test -t 114 -d f228b299-e42c-40f0-a67a-08786bb9b1a7 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:48 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/226f2859-e361-4f6c-a4d1-2f9f98291ec3 {"id":"226f2859-e361-4f6c-a4d1-2f9f98291ec3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754245} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13206254903965006868,"high":5483312206236578955},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375875505646,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.417314802+08:00","start-ts":457114710610018305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711855202311} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114712209096711,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/226f2859-e361-4f6c-a4d1-2f9f98291ec3 {"id":"226f2859-e361-4f6c-a4d1-2f9f98291ec3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754245} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13206254903965006868,"high":5483312206236578955},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375875505646,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.417314802+08:00","start-ts":457114710610018305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711855202311} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114712209096711,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/226f2859-e361-4f6c-a4d1-2f9f98291ec3 {"id":"226f2859-e361-4f6c-a4d1-2f9f98291ec3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754245} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13206254903965006868,"high":5483312206236578955},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375875505646,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.417314802+08:00","start-ts":457114710610018305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711855202311} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114712209096711,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:48 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1591915921.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.cli.16328.out cli tso query --pd=http://127.0.0.1:2379 command: check_safepoint_forward http://127.0.0.1:2379 7489367088931747056 457114699174772735 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.15627.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38675480-1c67-4771-89f7-fd32ef3245a3 {"id":"38675480-1c67-4771-89f7-fd32ef3245a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114712347246615,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38675480-1c67-4771-89f7-fd32ef3245a3 {"id":"38675480-1c67-4771-89f7-fd32ef3245a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114712347246615,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38675480-1c67-4771-89f7-fd32ef3245a3 {"id":"38675480-1c67-4771-89f7-fd32ef3245a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114712347246615,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:49 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=pause' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.2770727709.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1edf495e-fc6d-4190-8aa0-26efda5edc6d {"id":"1edf495e-fc6d-4190-8aa0-26efda5edc6d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754231} /tidb/cdc/default/__cdc_meta__/capture/68265de1-f3f5-41d9-8f78-f60b018a39cc {"id":"68265de1-f3f5-41d9-8f78-f60b018a39cc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be7c8 68265de1-f3f5-41d9-8f78-f60b018a39cc /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be7c8 68265de1-f3f5-41d9-8f78-f60b018a39cc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10037357187106854639,"high":7309533088047575176},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367299086423050,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:23.47402767+08:00","start-ts":457114705916592130,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707148668939} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114713046908933,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1edf495e-fc6d-4190-8aa0-26efda5edc6d {"id":"1edf495e-fc6d-4190-8aa0-26efda5edc6d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754231} /tidb/cdc/default/__cdc_meta__/capture/68265de1-f3f5-41d9-8f78-f60b018a39cc {"id":"68265de1-f3f5-41d9-8f78-f60b018a39cc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be7c8 68265de1-f3f5-41d9-8f78-f60b018a39cc /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be7c8 68265de1-f3f5-41d9-8f78-f60b018a39cc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10037357187106854639,"high":7309533088047575176},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367299086423050,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:23.47402767+08:00","start-ts":457114705916592130,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707148668939} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114713046908933,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1edf495e-fc6d-4190-8aa0-26efda5edc6d {"id":"1edf495e-fc6d-4190-8aa0-26efda5edc6d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754231} /tidb/cdc/default/__cdc_meta__/capture/68265de1-f3f5-41d9-8f78-f60b018a39cc {"id":"68265de1-f3f5-41d9-8f78-f60b018a39cc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99be7c8 68265de1-f3f5-41d9-8f78-f60b018a39cc /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be529 1edf495e-fc6d-4190-8aa0-26efda5edc6d /tidb/cdc/default/__cdc_meta__/owner/223195ffd99be7c8 68265de1-f3f5-41d9-8f78-f60b018a39cc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10037357187106854639,"high":7309533088047575176},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367299086423050,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:23.47402767+08:00","start-ts":457114705916592130,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707148668939} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114713046908933,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later safepoint 457114699174772735 is not forward run task failed 7-th time, retry later table fail_over_ddl_test.test2 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:50 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4384418d-d615-4dbf-890d-049542690036 {"id":"4384418d-d615-4dbf-890d-049542690036","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036 /tidb/cdc/default/__cdc_meta__/owner/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4384418d-d615-4dbf-890d-049542690036 {"id":"4384418d-d615-4dbf-890d-049542690036","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036 /tidb/cdc/default/__cdc_meta__/owner/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4384418d-d615-4dbf-890d-049542690036 {"id":"4384418d-d615-4dbf-890d-049542690036","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036 /tidb/cdc/default/__cdc_meta__/owner/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114712998150145 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.18588.out cli changefeed create --start-ts=457114712998150145 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367415843588156,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:50.507163816+08:00","start_ts":457114712998150145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114712998150145,"checkpoint_ts":457114712998150145,"checkpoint_time":"2025-04-04 16:10:45.751","gid":{"low":10972440859670203073,"high":14428803174907767988}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x + tso='457114713874759681 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114713874759681 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:50 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1635616358.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13743) - No such process wait process 13743 exit for 1-th time... process 13743 already exit [Fri Apr 4 16:10:50 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1385813860.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb27f5d0-c6d6-4351-933b-420ef052bbcf {"id":"eb27f5d0-c6d6-4351-933b-420ef052bbcf","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/capture/f5999f63-d209-462c-b50e-03c2d4b94130 {"id":"f5999f63-d209-462c-b50e-03c2d4b94130","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96d4e eb27f5d0-c6d6-4351-933b-420ef052bbcf /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96d4e eb27f5d0-c6d6-4351-933b-420ef052bbcf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4922064178680103469,"high":15056557442061542273},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367362330872194,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:38.758872607+08:00","start-ts":457114709914288129,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711159472135} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114711670652950,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb27f5d0-c6d6-4351-933b-420ef052bbcf {"id":"eb27f5d0-c6d6-4351-933b-420ef052bbcf","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/capture/f5999f63-d209-462c-b50e-03c2d4b94130 {"id":"f5999f63-d209-462c-b50e-03c2d4b94130","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96d4e eb27f5d0-c6d6-4351-933b-420ef052bbcf /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96d4e eb27f5d0-c6d6-4351-933b-420ef052bbcf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4922064178680103469,"high":15056557442061542273},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367362330872194,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:38.758872607+08:00","start-ts":457114709914288129,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711159472135} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114711670652950,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb27f5d0-c6d6-4351-933b-420ef052bbcf {"id":"eb27f5d0-c6d6-4351-933b-420ef052bbcf","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/capture/f5999f63-d209-462c-b50e-03c2d4b94130 {"id":"f5999f63-d209-462c-b50e-03c2d4b94130","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96be2 b7057dfe-85fb-44f3-aae3-a11ca8b973f0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96d4e eb27f5d0-c6d6-4351-933b-420ef052bbcf /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96d4e eb27f5d0-c6d6-4351-933b-420ef052bbcf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4922064178680103469,"high":15056557442061542273},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367362330872194,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:38.758872607+08:00","start-ts":457114709914288129,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711159472135} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114711670652950,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.13441.out cli capture list Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='457114714069008386 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114714069008386 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:10:51 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1566615668.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x Command succeeded on attempt 1 check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:51 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c0ce219-838a-4059-b526-aaaee22e8692 {"id":"1c0ce219-838a-4059-b526-aaaee22e8692","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754248} /tidb/cdc/default/__cdc_meta__/capture/226f2859-e361-4f6c-a4d1-2f9f98291ec3 {"id":"226f2859-e361-4f6c-a4d1-2f9f98291ec3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754245} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58c23 1c0ce219-838a-4059-b526-aaaee22e8692 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58c23 1c0ce219-838a-4059-b526-aaaee22e8692 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13206254903965006868,"high":5483312206236578955},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375875505646,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.417314802+08:00","start-ts":457114710610018305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711855202311} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714044104716,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c0ce219-838a-4059-b526-aaaee22e8692 {"id":"1c0ce219-838a-4059-b526-aaaee22e8692","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754248} /tidb/cdc/default/__cdc_meta__/capture/226f2859-e361-4f6c-a4d1-2f9f98291ec3 {"id":"226f2859-e361-4f6c-a4d1-2f9f98291ec3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754245} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58c23 1c0ce219-838a-4059-b526-aaaee22e8692 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58c23 1c0ce219-838a-4059-b526-aaaee22e8692 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13206254903965006868,"high":5483312206236578955},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375875505646,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.417314802+08:00","start-ts":457114710610018305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711855202311} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714044104716,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c0ce219-838a-4059-b526-aaaee22e8692 {"id":"1c0ce219-838a-4059-b526-aaaee22e8692","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754248} /tidb/cdc/default/__cdc_meta__/capture/226f2859-e361-4f6c-a4d1-2f9f98291ec3 {"id":"226f2859-e361-4f6c-a4d1-2f9f98291ec3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754245} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58ae1 c715cf08-ed13-42c6-a971-254e75f1136c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58c23 1c0ce219-838a-4059-b526-aaaee22e8692 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58c23 1c0ce219-838a-4059-b526-aaaee22e8692 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13206254903965006868,"high":5483312206236578955},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375875505646,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.417314802+08:00","start-ts":457114710610018305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711855202311} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714044104716,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.15998.out cli capture list check diff failed 2-th time, retry later command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully table fail_over_ddl_test.test2 not exists for 1-th check, retry later + set +x + ans='[ { "id": "eb27f5d0-c6d6-4351-933b-420ef052bbcf", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" }, { "id": "f5999f63-d209-462c-b50e-03c2d4b94130", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"eb27f5d0-c6d6-4351-933b-420ef052bbcf",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '},' '{' '"id":' '"f5999f63-d209-462c-b50e-03c2d4b94130",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=eb27f5d0-c6d6-4351-933b-420ef052bbcf + '[' -z eb27f5d0-c6d6-4351-933b-420ef052bbcf ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d eb27f5d0-c6d6-4351-933b-420ef052bbcf + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.13479.out cli changefeed move-table -c test -t 114 -d eb27f5d0-c6d6-4351-933b-420ef052bbcf cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseN-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:50 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.1163511637.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + ans='[ { "id": "226f2859-e361-4f6c-a4d1-2f9f98291ec3", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "1c0ce219-838a-4059-b526-aaaee22e8692", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"226f2859-e361-4f6c-a4d1-2f9f98291ec3",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"1c0ce219-838a-4059-b526-aaaee22e8692",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=1c0ce219-838a-4059-b526-aaaee22e8692 + '[' -z 1c0ce219-838a-4059-b526-aaaee22e8692 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 1c0ce219-838a-4059-b526-aaaee22e8692 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.16028.out cli changefeed move-table -c test -t 114 -d 1c0ce219-838a-4059-b526-aaaee22e8692 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38675480-1c67-4771-89f7-fd32ef3245a3 {"id":"38675480-1c67-4771-89f7-fd32ef3245a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/capture/428392c8-1ab9-4993-aa03-1b32e552f004 {"id":"428392c8-1ab9-4993-aa03-1b32e552f004","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754250} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e35153 428392c8-1ab9-4993-aa03-1b32e552f004 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e35153 428392c8-1ab9-4993-aa03-1b32e552f004 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114712347246615,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38675480-1c67-4771-89f7-fd32ef3245a3 {"id":"38675480-1c67-4771-89f7-fd32ef3245a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/capture/428392c8-1ab9-4993-aa03-1b32e552f004 {"id":"428392c8-1ab9-4993-aa03-1b32e552f004","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754250} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e35153 428392c8-1ab9-4993-aa03-1b32e552f004 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e35153 428392c8-1ab9-4993-aa03-1b32e552f004 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114712347246615,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38675480-1c67-4771-89f7-fd32ef3245a3 {"id":"38675480-1c67-4771-89f7-fd32ef3245a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754247} /tidb/cdc/default/__cdc_meta__/capture/428392c8-1ab9-4993-aa03-1b32e552f004 {"id":"428392c8-1ab9-4993-aa03-1b32e552f004","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754250} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e34fe4 4e401c8b-6a15-4796-85b3-01160932b36f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e35153 428392c8-1ab9-4993-aa03-1b32e552f004 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e35153 428392c8-1ab9-4993-aa03-1b32e552f004 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114712347246615,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.27776.out cli capture list command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:53 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1f6983e-fd91-440a-a05c-8e3642fb4455 {"id":"a1f6983e-fd91-440a-a05c-8e3642fb4455","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754250} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1f6983e-fd91-440a-a05c-8e3642fb4455 {"id":"a1f6983e-fd91-440a-a05c-8e3642fb4455","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754250} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1f6983e-fd91-440a-a05c-8e3642fb4455 {"id":"a1f6983e-fd91-440a-a05c-8e3642fb4455","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754250} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114713874759681 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.cli.16414.out cli changefeed create --start-ts=457114713874759681 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:53 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/86683488-78bb-43da-b386-caecc744894a {"id":"86683488-78bb-43da-b386-caecc744894a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/capture/8921c6c6-7121-427c-a7da-4d9d9a4f2c66 {"id":"8921c6c6-7121-427c-a7da-4d9d9a4f2c66","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754210} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9463879 86683488-78bb-43da-b386-caecc744894a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9463879 86683488-78bb-43da-b386-caecc744894a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7658969282484924454,"high":9210270466288278959},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367202340662815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:01.515468822+08:00","start-ts":457114700151783425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701397229575} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114705460199429,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/86683488-78bb-43da-b386-caecc744894a {"id":"86683488-78bb-43da-b386-caecc744894a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/capture/8921c6c6-7121-427c-a7da-4d9d9a4f2c66 {"id":"8921c6c6-7121-427c-a7da-4d9d9a4f2c66","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754210} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9463879 86683488-78bb-43da-b386-caecc744894a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9463879 86683488-78bb-43da-b386-caecc744894a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7658969282484924454,"high":9210270466288278959},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367202340662815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:01.515468822+08:00","start-ts":457114700151783425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701397229575} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114705460199429,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/86683488-78bb-43da-b386-caecc744894a {"id":"86683488-78bb-43da-b386-caecc744894a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/capture/8921c6c6-7121-427c-a7da-4d9d9a4f2c66 {"id":"8921c6c6-7121-427c-a7da-4d9d9a4f2c66","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754210} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9463879 86683488-78bb-43da-b386-caecc744894a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9463350 8921c6c6-7121-427c-a7da-4d9d9a4f2c66 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9463879 86683488-78bb-43da-b386-caecc744894a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7658969282484924454,"high":9210270466288278959},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367202340662815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:01.515468822+08:00","start-ts":457114700151783425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114701397229575} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114705460199429,"progress":1}' + grep -q 'etcd info' + break + set +x table fail_over_ddl_test.test2 exists Create changefeed successfully! ID: test Info: {"upstream_id":7489367424752522402,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:53.902309544+08:00","start_ts":457114713874759681,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114713874759681,"checkpoint_ts":457114713874759681,"checkpoint_time":"2025-04-04 16:10:49.095","gid":{"low":10685424696973760219,"high":2152855705924863395}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:53 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 {"id":"3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114713672646683,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 {"id":"3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114713672646683,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 {"id":"3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114713672646683,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:10:54 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1392513927.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:54 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da5ae180-2324-4aa3-86cb-1652b2444f24 {"id":"da5ae180-2324-4aa3-86cb-1652b2444f24","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da5ae180-2324-4aa3-86cb-1652b2444f24 {"id":"da5ae180-2324-4aa3-86cb-1652b2444f24","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da5ae180-2324-4aa3-86cb-1652b2444f24 {"id":"da5ae180-2324-4aa3-86cb-1652b2444f24","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114714069008386 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.15729.out cli changefeed create --start-ts=457114714069008386 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367431094912519,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:10:54.607731103+08:00","start_ts":457114714069008386,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114714069008386,"checkpoint_ts":457114714069008386,"checkpoint_time":"2025-04-04 16:10:49.836","gid":{"low":9389614194233293602,"high":17585974546411942538}} PASS /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18523) - No such process wait process 18523 exit for 1-th time... process 18523 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:10:54 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1865318655.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=3 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' 3 == -1 ']' + '[' 3 == 3 ']' + exit 0 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x + ans='[ { "id": "38675480-1c67-4771-89f7-fd32ef3245a3", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "428392c8-1ab9-4993-aa03-1b32e552f004", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"38675480-1c67-4771-89f7-fd32ef3245a3",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"428392c8-1ab9-4993-aa03-1b32e552f004",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=428392c8-1ab9-4993-aa03-1b32e552f004 + '[' -z 428392c8-1ab9-4993-aa03-1b32e552f004 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 428392c8-1ab9-4993-aa03-1b32e552f004 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.27804.out cli changefeed move-table -c test -t 114 -d 428392c8-1ab9-4993-aa03-1b32e552f004 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseB-1 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later [2025/04/04 16:10:55.631 +08:00] [DEBUG] [peer.rs:2330] ["failed to schedule peer tick"] [err="sending on a disconnected channel"] [tick=CheckLeaderLease] [peer_id=79] [region_id=78] [thread_id=81] check diff failed 1-th time, retry later + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367466750404009 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6986e80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:14533, start at 2025-04-04 16:10:56.836028232 +0800 CST m=+3.751045566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:56.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:56.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:56.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367466750404009 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6986e80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:14533, start at 2025-04-04 16:10:56.836028232 +0800 CST m=+3.751045566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:56.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:56.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:56.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13701) - No such process wait process 13701 exit for 1-th time... process 13701 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13772) - No such process wait process 13772 exit for 1-th time... process 13772 already exit [Fri Apr 4 16:10:56 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1389413896.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 {"id":"3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/capture/bce8a41c-a4de-478b-9579-00a64999d463 {"id":"bce8a41c-a4de-478b-9579-00a64999d463","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54d26 bce8a41c-a4de-478b-9579-00a64999d463 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54d26 bce8a41c-a4de-478b-9579-00a64999d463 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114713672646683,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 {"id":"3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/capture/bce8a41c-a4de-478b-9579-00a64999d463 {"id":"bce8a41c-a4de-478b-9579-00a64999d463","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54d26 bce8a41c-a4de-478b-9579-00a64999d463 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54d26 bce8a41c-a4de-478b-9579-00a64999d463 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114713672646683,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 {"id":"3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754251} /tidb/cdc/default/__cdc_meta__/capture/bce8a41c-a4de-478b-9579-00a64999d463 {"id":"bce8a41c-a4de-478b-9579-00a64999d463","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54bea 7f5cd251-7118-4a06-abe8-830d59e3fd7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54d26 bce8a41c-a4de-478b-9579-00a64999d463 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f54d26 bce8a41c-a4de-478b-9579-00a64999d463 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114713672646683,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367474512564333 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6988d8000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:14602, start at 2025-04-04 16:10:56.969850039 +0800 CST m=+3.721306396 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:12:57.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:10:57.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:00:57.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3087 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cc294bc-3bc8-4688-a3c9-3cf756e2d677 {"id":"2cc294bc-3bc8-4688-a3c9-3cf756e2d677","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/owner/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10972440859670203073,"high":14428803174907767988},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367415843588156,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:50.507163816+08:00","start-ts":457114712998150145,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114714243334149} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714597228560,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cc294bc-3bc8-4688-a3c9-3cf756e2d677 {"id":"2cc294bc-3bc8-4688-a3c9-3cf756e2d677","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/owner/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10972440859670203073,"high":14428803174907767988},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367415843588156,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:50.507163816+08:00","start-ts":457114712998150145,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114714243334149} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714597228560,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cc294bc-3bc8-4688-a3c9-3cf756e2d677 {"id":"2cc294bc-3bc8-4688-a3c9-3cf756e2d677","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/owner/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10972440859670203073,"high":14428803174907767988},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367415843588156,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:50.507163816+08:00","start-ts":457114712998150145,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114714243334149} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714597228560,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:57 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1873018732.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16361) - No such process wait process 16361 exit for 1-th time... process 16361 already exit [Fri Apr 4 16:10:57 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1649016492.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15671) - No such process wait process 15671 exit for 1-th time... process 15671 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:10:58 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1580715809.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:10:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40b46c1d-ad72-4ce5-a015-da7dbe6dd37c {"id":"40b46c1d-ad72-4ce5-a015-da7dbe6dd37c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754256} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715028979715,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40b46c1d-ad72-4ce5-a015-da7dbe6dd37c {"id":"40b46c1d-ad72-4ce5-a015-da7dbe6dd37c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754256} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715028979715,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40b46c1d-ad72-4ce5-a015-da7dbe6dd37c {"id":"40b46c1d-ad72-4ce5-a015-da7dbe6dd37c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754256} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715028979715,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:10:59 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1397113973.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseH-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.cli.15886.out cli tso query --pd=http://127.0.0.1:2379 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_N Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3668 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cc294bc-3bc8-4688-a3c9-3cf756e2d677 {"id":"2cc294bc-3bc8-4688-a3c9-3cf756e2d677","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/capture/847f3200-95bc-497d-b7b2-38cbeeaf0e9e {"id":"847f3200-95bc-497d-b7b2-38cbeeaf0e9e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754258} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda073027 847f3200-95bc-497d-b7b2-38cbeeaf0e9e /tidb/cdc/default/__cdc_meta__/owner/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/owner/223195ffda073027 847f3200-95bc-497d-b7b2-38cbeeaf0e9e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10972440859670203073,"high":14428803174907767988},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367415843588156,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:50.507163816+08:00","start-ts":457114712998150145,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114714243334149} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714623442964,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cc294bc-3bc8-4688-a3c9-3cf756e2d677 {"id":"2cc294bc-3bc8-4688-a3c9-3cf756e2d677","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/capture/847f3200-95bc-497d-b7b2-38cbeeaf0e9e {"id":"847f3200-95bc-497d-b7b2-38cbeeaf0e9e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754258} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda073027 847f3200-95bc-497d-b7b2-38cbeeaf0e9e /tidb/cdc/default/__cdc_meta__/owner/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/owner/223195ffda073027 847f3200-95bc-497d-b7b2-38cbeeaf0e9e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10972440859670203073,"high":14428803174907767988},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367415843588156,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:50.507163816+08:00","start-ts":457114712998150145,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114714243334149} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714623442964,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cc294bc-3bc8-4688-a3c9-3cf756e2d677 {"id":"2cc294bc-3bc8-4688-a3c9-3cf756e2d677","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/capture/847f3200-95bc-497d-b7b2-38cbeeaf0e9e {"id":"847f3200-95bc-497d-b7b2-38cbeeaf0e9e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754258} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072ee5 4384418d-d615-4dbf-890d-049542690036 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda073027 847f3200-95bc-497d-b7b2-38cbeeaf0e9e /tidb/cdc/default/__cdc_meta__/owner/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/owner/223195ffda073027 847f3200-95bc-497d-b7b2-38cbeeaf0e9e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10972440859670203073,"high":14428803174907767988},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367415843588156,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:50.507163816+08:00","start-ts":457114712998150145,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114714243334149} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714623442964,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.18805.out cli capture list + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22cdb486-c505-4001-8361-36481330a919 {"id":"22cdb486-c505-4001-8361-36481330a919","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754258} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715513421837,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22cdb486-c505-4001-8361-36481330a919 {"id":"22cdb486-c505-4001-8361-36481330a919","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754258} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715513421837,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22cdb486-c505-4001-8361-36481330a919 {"id":"22cdb486-c505-4001-8361-36481330a919","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754258} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715513421837,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:11:01 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1656316565.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16064) - No such process wait process 16064 exit for 1-th time... process 16064 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + set +x + tso='457114716758081537 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114716758081537 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:01 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1592915931.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4108600-ea15-4ade-a23f-723deb24af56 {"id":"a4108600-ea15-4ade-a23f-723deb24af56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9389614194233293602,"high":17585974546411942538},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367431094912519,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:54.607731103+08:00","start-ts":457114714069008386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715313930248} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715694039080,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4108600-ea15-4ade-a23f-723deb24af56 {"id":"a4108600-ea15-4ade-a23f-723deb24af56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9389614194233293602,"high":17585974546411942538},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367431094912519,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:54.607731103+08:00","start-ts":457114714069008386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715313930248} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715694039080,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4108600-ea15-4ade-a23f-723deb24af56 {"id":"a4108600-ea15-4ade-a23f-723deb24af56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9389614194233293602,"high":17585974546411942538},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367431094912519,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:54.607731103+08:00","start-ts":457114714069008386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715313930248} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715694039080,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:01 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1588315885.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + ans='[ { "id": "2cc294bc-3bc8-4688-a3c9-3cf756e2d677", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "847f3200-95bc-497d-b7b2-38cbeeaf0e9e", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"2cc294bc-3bc8-4688-a3c9-3cf756e2d677",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"847f3200-95bc-497d-b7b2-38cbeeaf0e9e",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=847f3200-95bc-497d-b7b2-38cbeeaf0e9e + '[' -z 847f3200-95bc-497d-b7b2-38cbeeaf0e9e ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 847f3200-95bc-497d-b7b2-38cbeeaf0e9e + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.cli.18834.out cli changefeed move-table -c test -t 114 -d 847f3200-95bc-497d-b7b2-38cbeeaf0e9e + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40b46c1d-ad72-4ce5-a015-da7dbe6dd37c {"id":"40b46c1d-ad72-4ce5-a015-da7dbe6dd37c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754256} /tidb/cdc/default/__cdc_meta__/capture/74149a88-acee-485d-b51d-a6c608f78e11 {"id":"74149a88-acee-485d-b51d-a6c608f78e11","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fdf02 74149a88-acee-485d-b51d-a6c608f78e11 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fdf02 74149a88-acee-485d-b51d-a6c608f78e11 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715028979715,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40b46c1d-ad72-4ce5-a015-da7dbe6dd37c {"id":"40b46c1d-ad72-4ce5-a015-da7dbe6dd37c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754256} /tidb/cdc/default/__cdc_meta__/capture/74149a88-acee-485d-b51d-a6c608f78e11 {"id":"74149a88-acee-485d-b51d-a6c608f78e11","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fdf02 74149a88-acee-485d-b51d-a6c608f78e11 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fdf02 74149a88-acee-485d-b51d-a6c608f78e11 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715028979715,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40b46c1d-ad72-4ce5-a015-da7dbe6dd37c {"id":"40b46c1d-ad72-4ce5-a015-da7dbe6dd37c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754256} /tidb/cdc/default/__cdc_meta__/capture/74149a88-acee-485d-b51d-a6c608f78e11 {"id":"74149a88-acee-485d-b51d-a6c608f78e11","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fd9c3 9fdae1a7-65b0-45e1-8331-ae2a90acfddf /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fdf02 74149a88-acee-485d-b51d-a6c608f78e11 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fde9b 40b46c1d-ad72-4ce5-a015-da7dbe6dd37c /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fdf02 74149a88-acee-485d-b51d-a6c608f78e11 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3769471121336011646,"high":13664478006238333864},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367304856166827,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.057376155+08:00","start-ts":457114706325798913,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707570982917} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715028979715,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully check diff failed 1-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_B Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13319) - No such process wait process 13319 exit for 1-th time... process 13319 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22cdb486-c505-4001-8361-36481330a919 {"id":"22cdb486-c505-4001-8361-36481330a919","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754258} /tidb/cdc/default/__cdc_meta__/capture/c3619d94-168d-4a47-b035-bac8a85ea30b {"id":"c3619d94-168d-4a47-b035-bac8a85ea30b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754261} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda105727 c3619d94-168d-4a47-b035-bac8a85ea30b /tidb/cdc/default/__cdc_meta__/owner/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/owner/223195ffda105727 c3619d94-168d-4a47-b035-bac8a85ea30b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715513421837,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22cdb486-c505-4001-8361-36481330a919 {"id":"22cdb486-c505-4001-8361-36481330a919","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754258} /tidb/cdc/default/__cdc_meta__/capture/c3619d94-168d-4a47-b035-bac8a85ea30b {"id":"c3619d94-168d-4a47-b035-bac8a85ea30b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754261} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda105727 c3619d94-168d-4a47-b035-bac8a85ea30b /tidb/cdc/default/__cdc_meta__/owner/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/owner/223195ffda105727 c3619d94-168d-4a47-b035-bac8a85ea30b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test + grep -q 'failed to get info:' {"checkpoint-ts":457114715513421837,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22cdb486-c505-4001-8361-36481330a919 {"id":"22cdb486-c505-4001-8361-36481330a919","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754258} /tidb/cdc/default/__cdc_meta__/capture/c3619d94-168d-4a47-b035-bac8a85ea30b {"id":"c3619d94-168d-4a47-b035-bac8a85ea30b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754261} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1055e5 a1f6983e-fd91-440a-a05c-8e3642fb4455 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda105727 c3619d94-168d-4a47-b035-bac8a85ea30b /tidb/cdc/default/__cdc_meta__/owner/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/owner/223195ffda105727 c3619d94-168d-4a47-b035-bac8a85ea30b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715513421837,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:04 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4509bda9-dde6-41f1-8dcf-77d34e671e4d {"id":"4509bda9-dde6-41f1-8dcf-77d34e671e4d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754261} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4509bda9-dde6-41f1-8dcf-77d34e671e4d {"id":"4509bda9-dde6-41f1-8dcf-77d34e671e4d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754261} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4509bda9-dde6-41f1-8dcf-77d34e671e4d {"id":"4509bda9-dde6-41f1-8dcf-77d34e671e4d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754261} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114716758081537 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.cli.15991.out cli changefeed create --start-ts=457114716758081537 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367466750404009,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:04.873575671+08:00","start_ts":457114716758081537,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114716758081537,"checkpoint_ts":457114716758081537,"checkpoint_time":"2025-04-04 16:11:00.094","gid":{"low":17458474664049934162,"high":8672975050330466229}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x command: check_safepoint_forward http://127.0.0.1:2379 7489367088931747056 457114699174772735 run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f {"id":"5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754262} /tidb/cdc/default/__cdc_meta__/capture/a4108600-ea15-4ade-a23f-723deb24af56 {"id":"a4108600-ea15-4ade-a23f-723deb24af56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda175228 5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f /tidb/cdc/default/__cdc_meta__/owner/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/owner/223195ffda175228 5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9389614194233293602,"high":17585974546411942538},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367431094912519,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:54.607731103+08:00","start-ts":457114714069008386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715313930248} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715694039080,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f {"id":"5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754262} /tidb/cdc/default/__cdc_meta__/capture/a4108600-ea15-4ade-a23f-723deb24af56 {"id":"a4108600-ea15-4ade-a23f-723deb24af56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda175228 5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f /tidb/cdc/default/__cdc_meta__/owner/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/owner/223195ffda175228 5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9389614194233293602,"high":17585974546411942538},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367431094912519,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:54.607731103+08:00","start-ts":457114714069008386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715313930248} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715694039080,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f {"id":"5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754262} /tidb/cdc/default/__cdc_meta__/capture/a4108600-ea15-4ade-a23f-723deb24af56 {"id":"a4108600-ea15-4ade-a23f-723deb24af56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1750e4 da5ae180-2324-4aa3-86cb-1652b2444f24 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda175228 5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f /tidb/cdc/default/__cdc_meta__/owner/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/owner/223195ffda175228 5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9389614194233293602,"high":17585974546411942538},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367431094912519,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:54.607731103+08:00","start-ts":457114714069008386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715313930248} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715694039080,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.15954.out cli capture list Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 2-th time, retry later command: check_changefeed_state http://127.0.0.1:2379 1752807653410833484610169415565702060686 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=1752807653410833484610169415565702060686 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c 1752807653410833484610169415565702060686 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489367088931747056, "id": "1752807653410833484610169415565702060686", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114717577019394, "checkpoint_time": "2025-04-04 16:11:03.218", "error": null }' + echo '{ "upstream_id": 7489367088931747056, "id": "1752807653410833484610169415565702060686", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114717577019394, "checkpoint_time": "2025-04-04 16:11:03.218", "error": null }' { "upstream_id": 7489367088931747056, "id": "1752807653410833484610169415565702060686", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114717577019394, "checkpoint_time": "2025-04-04 16:11:03.218", "error": null } ++ echo '{' '"upstream_id":' 7489367088931747056, '"id":' '"1752807653410833484610169415565702060686",' '"namespace":' '"default",' '"state":' '"stopped",' '"checkpoint_tso":' 457114717577019394, '"checkpoint_time":' '"2025-04-04' '16:11:03.218",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489367088931747056, '"id":' '"1752807653410833484610169415565702060686",' '"namespace":' '"default",' '"state":' '"stopped",' '"checkpoint_tso":' 457114717577019394, '"checkpoint_time":' '"2025-04-04' '16:11:03.218",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully command: check_safepoint_equal http://127.0.0.1:2379 7489367088931747056 Starting Upstream TiDB... + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:05 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1633116333.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Command succeeded on attempt 1 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x + ans='[ { "id": "a4108600-ea15-4ade-a23f-723deb24af56", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"a4108600-ea15-4ade-a23f-723deb24af56",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f + '[' -z 5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.15983.out cli changefeed move-table -c test -t 114 -d 5eebf72b-377a-4bf6-b3ef-8da3ba85bf8f + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseD-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_H Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table fail_over_ddl_test.test2 not exists for 1-th check, retry later command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully run task successfully command: check_changefeed_state http://127.0.0.1:2379 1752807653410833484610169415565702060686 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=1752807653410833484610169415565702060686 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c 1752807653410833484610169415565702060686 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489367088931747056, "id": "1752807653410833484610169415565702060686", "namespace": "default", "state": "normal", "checkpoint_tso": 457114717577019394, "checkpoint_time": "2025-04-04 16:11:03.218", "error": null }' + echo '{ "upstream_id": 7489367088931747056, "id": "1752807653410833484610169415565702060686", "namespace": "default", "state": "normal", "checkpoint_tso": 457114717577019394, "checkpoint_time": "2025-04-04 16:11:03.218", "error": null }' { "upstream_id": 7489367088931747056, "id": "1752807653410833484610169415565702060686", "namespace": "default", "state": "normal", "checkpoint_tso": 457114717577019394, "checkpoint_time": "2025-04-04 16:11:03.218", "error": null } ++ echo '{' '"upstream_id":' 7489367088931747056, '"id":' '"1752807653410833484610169415565702060686",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114717577019394, '"checkpoint_time":' '"2025-04-04' '16:11:03.218",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489367088931747056, '"id":' '"1752807653410833484610169415565702060686",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114717577019394, '"checkpoint_time":' '"2025-04-04' '16:11:03.218",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully command: check_safepoint_forward http://127.0.0.1:2379 7489367088931747056 457114714418446340 457114717577019394 + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":2}' + echo '{"count":2}' {"count":2} ++ echo '{"count":2}' ++ jq -r .count + value=2 + '[' -1 == -1 ']' + '[' 2 '!=' null ']' + exit 0 [Fri Apr 4 16:11:08 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1360813610.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1baf3297-5698-4bfc-8a2f-5d29437346fc {"id":"1baf3297-5698-4bfc-8a2f-5d29437346fc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/capture/5c36af1d-d7ac-4407-ae55-198512a40789 {"id":"5c36af1d-d7ac-4407-ae55-198512a40789","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754266} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fc21a 5c36af1d-d7ac-4407-ae55-198512a40789 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fc21a 5c36af1d-d7ac-4407-ae55-198512a40789 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4558862218431368628,"high":4669594375052180648},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367301609395499,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:24.536280375+08:00","start-ts":457114706186862593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707432046598} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114716187656201,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1baf3297-5698-4bfc-8a2f-5d29437346fc {"id":"1baf3297-5698-4bfc-8a2f-5d29437346fc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/capture/5c36af1d-d7ac-4407-ae55-198512a40789 {"id":"5c36af1d-d7ac-4407-ae55-198512a40789","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754266} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fc21a 5c36af1d-d7ac-4407-ae55-198512a40789 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fc21a 5c36af1d-d7ac-4407-ae55-198512a40789 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4558862218431368628,"high":4669594375052180648},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367301609395499,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:24.536280375+08:00","start-ts":457114706186862593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707432046598} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114716187656201,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1baf3297-5698-4bfc-8a2f-5d29437346fc {"id":"1baf3297-5698-4bfc-8a2f-5d29437346fc","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754232} /tidb/cdc/default/__cdc_meta__/capture/5c36af1d-d7ac-4407-ae55-198512a40789 {"id":"5c36af1d-d7ac-4407-ae55-198512a40789","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754266} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd99fc21a 5c36af1d-d7ac-4407-ae55-198512a40789 /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fbc2a 1baf3297-5698-4bfc-8a2f-5d29437346fc /tidb/cdc/default/__cdc_meta__/owner/223195ffd99fc21a 5c36af1d-d7ac-4407-ae55-198512a40789 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4558862218431368628,"high":4669594375052180648},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367301609395499,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:24.536280375+08:00","start-ts":457114706186862593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707432046598} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114716187656201,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16495) - No such process wait process 16495 exit for 1-th time... process 16495 already exit run task successfully check diff failed 1-th time, retry later command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16568) - No such process wait process 16568 exit for 1-th time... process 16568 already exit [Fri Apr 4 16:11:09 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1668116683.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18735) - No such process wait process 18735 exit for 1-th time... process 18735 already exit [Fri Apr 4 16:11:09 CST 2025] <<<<<< START cdc server in fail_over_ddl_C case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=sleep(30000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_C.1891218914.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_C/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_C/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table fail_over_ddl_test.test2 exists run task successfully command: check_changefeed_state http://127.0.0.1:2379 1752807653410833484610169415565702060686 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=1752807653410833484610169415565702060686 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c 1752807653410833484610169415565702060686 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489367088931747056, "id": "1752807653410833484610169415565702060686", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114717577019394, "checkpoint_time": "2025-04-04 16:11:03.218", "error": null }' + echo '{ "upstream_id": 7489367088931747056, "id": "1752807653410833484610169415565702060686", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114717577019394, "checkpoint_time": "2025-04-04 16:11:03.218", "error": null }' { "upstream_id": 7489367088931747056, "id": "1752807653410833484610169415565702060686", "namespace": "default", "state": "stopped", "checkpoint_tso": 457114717577019394, "checkpoint_time": "2025-04-04 16:11:03.218", "error": null } ++ echo '{' '"upstream_id":' 7489367088931747056, '"id":' '"1752807653410833484610169415565702060686",' '"namespace":' '"default",' '"state":' '"stopped",' '"checkpoint_tso":' 457114717577019394, '"checkpoint_time":' '"2025-04-04' '16:11:03.218",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489367088931747056, '"id":' '"1752807653410833484610169415565702060686",' '"namespace":' '"default",' '"state":' '"stopped",' '"checkpoint_tso":' 457114717577019394, '"checkpoint_time":' '"2025-04-04' '16:11:03.218",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully command: check_changefeed_state http://127.0.0.1:2379 151511097882765978997140042014189208459 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=151511097882765978997140042014189208459 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query -c 151511097882765978997140042014189208459 -s ++ grep -v 'Command to ticdc' + info='{ "upstream_id": 7489367088931747056, "id": "151511097882765978997140042014189208459", "namespace": "default", "state": "normal", "checkpoint_tso": 457114719359860739, "checkpoint_time": "2025-04-04 16:11:10.019", "error": null }' + echo '{ "upstream_id": 7489367088931747056, "id": "151511097882765978997140042014189208459", "namespace": "default", "state": "normal", "checkpoint_tso": 457114719359860739, "checkpoint_time": "2025-04-04 16:11:10.019", "error": null }' { "upstream_id": 7489367088931747056, "id": "151511097882765978997140042014189208459", "namespace": "default", "state": "normal", "checkpoint_tso": 457114719359860739, "checkpoint_time": "2025-04-04 16:11:10.019", "error": null } ++ echo '{' '"upstream_id":' 7489367088931747056, '"id":' '"151511097882765978997140042014189208459",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114719359860739, '"checkpoint_time":' '"2025-04-04' '16:11:10.019",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489367088931747056, '"id":' '"151511097882765978997140042014189208459",' '"namespace":' '"default",' '"state":' '"normal",' '"checkpoint_tso":' 457114719359860739, '"checkpoint_time":' '"2025-04-04' '16:11:10.019",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully command: check_safepoint_equal http://127.0.0.1:2379 7489367088931747056 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15934) - No such process wait process 15934 exit for 1-th time... process 15934 already exit [Fri Apr 4 16:11:10 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1608816090.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367534224812083 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6a51980012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:13785, start at 2025-04-04 16:11:09.842119752 +0800 CST m=+3.672780279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:09.852 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:09.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:09.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367534224812083 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6a51980012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:13785, start at 2025-04-04 16:11:09.842119752 +0800 CST m=+3.672780279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:09.852 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:09.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:09.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367531951198218 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6a523c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:13849, start at 2025-04-04 16:11:09.843152867 +0800 CST m=+3.513794245 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:09.853 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:09.839 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:09.839 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... safepoint is unexpected forward: 457114714418446340 457114719359860739 457114717577019394 -> 457114714418446340 457114717577019394 run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367536119683961 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6a56800009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:11161, start at 2025-04-04 16:11:10.137709751 +0800 CST m=+3.091118862 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:10.145 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:10.112 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:10.112 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367536119683961 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6a56800009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:11161, start at 2025-04-04 16:11:10.137709751 +0800 CST m=+3.091118862 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:10.145 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:10.112 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:10.112 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367532565835876 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6a583c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:11230, start at 2025-04-04 16:11:10.236426648 +0800 CST m=+3.074953833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:10.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:10.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:10.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13461) - No such process wait process 13461 exit for 1-th time... process 13461 already exit [Fri Apr 4 16:11:08 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1365713659.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78b69aa3-0140-46b5-bc0f-6c42ce922694 {"id":"78b69aa3-0140-46b5-bc0f-6c42ce922694","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754242} /tidb/cdc/default/__cdc_meta__/capture/da7377e7-7a8f-4402-b338-19a7f435d8c9 {"id":"da7377e7-7a8f-4402-b338-19a7f435d8c9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754268} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a806d0 da7377e7-7a8f-4402-b338-19a7f435d8c9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a806d0 da7377e7-7a8f-4402-b338-19a7f435d8c9 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6935560052145240848,"high":2121570029465933459},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367305182583740,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.100357698+08:00","start-ts":457114706335498241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707580682246} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114716349399051,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78b69aa3-0140-46b5-bc0f-6c42ce922694 {"id":"78b69aa3-0140-46b5-bc0f-6c42ce922694","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754242} /tidb/cdc/default/__cdc_meta__/capture/da7377e7-7a8f-4402-b338-19a7f435d8c9 {"id":"da7377e7-7a8f-4402-b338-19a7f435d8c9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754268} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a806d0 da7377e7-7a8f-4402-b338-19a7f435d8c9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a806d0 da7377e7-7a8f-4402-b338-19a7f435d8c9 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6935560052145240848,"high":2121570029465933459},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367305182583740,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.100357698+08:00","start-ts":457114706335498241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707580682246} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114716349399051,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78b69aa3-0140-46b5-bc0f-6c42ce922694 {"id":"78b69aa3-0140-46b5-bc0f-6c42ce922694","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754242} /tidb/cdc/default/__cdc_meta__/capture/da7377e7-7a8f-4402-b338-19a7f435d8c9 {"id":"da7377e7-7a8f-4402-b338-19a7f435d8c9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754268} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9a806d0 da7377e7-7a8f-4402-b338-19a7f435d8c9 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a8025c 78b69aa3-0140-46b5-bc0f-6c42ce922694 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9a806d0 da7377e7-7a8f-4402-b338-19a7f435d8c9 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":6935560052145240848,"high":2121570029465933459},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367305182583740,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:25.100357698+08:00","start-ts":457114706335498241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114707580682246} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114716349399051,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51b854d5-378c-4cad-9dad-3e7ad4a4202e {"id":"51b854d5-378c-4cad-9dad-3e7ad4a4202e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754241} /tidb/cdc/default/__cdc_meta__/capture/87a4f91f-83ab-4ca8-a467-234cb6921694 {"id":"87a4f91f-83ab-4ca8-a467-234cb6921694","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754269} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd64ed 87a4f91f-83ab-4ca8-a467-234cb6921694 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd64ed 87a4f91f-83ab-4ca8-a467-234cb6921694 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":237186811318878080,"high":15219283008584226746},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367355570179202,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:36.672698071+08:00","start-ts":457114709381087233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114710613426182} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718857854979,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51b854d5-378c-4cad-9dad-3e7ad4a4202e {"id":"51b854d5-378c-4cad-9dad-3e7ad4a4202e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754241} /tidb/cdc/default/__cdc_meta__/capture/87a4f91f-83ab-4ca8-a467-234cb6921694 {"id":"87a4f91f-83ab-4ca8-a467-234cb6921694","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754269} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd64ed 87a4f91f-83ab-4ca8-a467-234cb6921694 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd64ed 87a4f91f-83ab-4ca8-a467-234cb6921694 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":237186811318878080,"high":15219283008584226746},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367355570179202,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:36.672698071+08:00","start-ts":457114709381087233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114710613426182} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718857854979,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51b854d5-378c-4cad-9dad-3e7ad4a4202e {"id":"51b854d5-378c-4cad-9dad-3e7ad4a4202e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754241} /tidb/cdc/default/__cdc_meta__/capture/87a4f91f-83ab-4ca8-a467-234cb6921694 {"id":"87a4f91f-83ab-4ca8-a467-234cb6921694","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754269} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9cd64ed 87a4f91f-83ab-4ca8-a467-234cb6921694 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd5fc4 51b854d5-378c-4cad-9dad-3e7ad4a4202e /tidb/cdc/default/__cdc_meta__/owner/223195ffd9cd64ed 87a4f91f-83ab-4ca8-a467-234cb6921694 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":237186811318878080,"high":15219283008584226746},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367355570179202,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:36.672698071+08:00","start-ts":457114709381087233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114710613426182} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114718857854979,"progress":1}' + break + set +x check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d61c8b00-ea01-4aad-9d93-349c5dfa7bcc {"id":"d61c8b00-ea01-4aad-9d93-349c5dfa7bcc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754269} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/__cdc_meta__/owner/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114717846241291,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d61c8b00-ea01-4aad-9d93-349c5dfa7bcc {"id":"d61c8b00-ea01-4aad-9d93-349c5dfa7bcc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754269} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/__cdc_meta__/owner/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114717846241291,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d61c8b00-ea01-4aad-9d93-349c5dfa7bcc {"id":"d61c8b00-ea01-4aad-9d93-349c5dfa7bcc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754269} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/__cdc_meta__/owner/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114717846241291,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:12 CST 2025] <<<<<< START cdc server in fail_over_ddl_G case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_G.1674916751.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_G/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_G/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.15124.out cli tso query --pd=http://127.0.0.1:2379 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13372) - No such process wait process 13372 exit for 1-th time... process 13372 already exit [Fri Apr 4 16:11:10 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1355013552.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3565 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09e18413-4d6a-4b4c-a708-67eecbe0073b {"id":"09e18413-4d6a-4b4c-a708-67eecbe0073b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754270} /tidb/cdc/default/__cdc_meta__/capture/2cc294bc-3bc8-4688-a3c9-3cf756e2d677 {"id":"2cc294bc-3bc8-4688-a3c9-3cf756e2d677","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda07326f 09e18413-4d6a-4b4c-a708-67eecbe0073b /tidb/cdc/default/__cdc_meta__/owner/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/owner/223195ffda07326f 09e18413-4d6a-4b4c-a708-67eecbe0073b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10972440859670203073,"high":14428803174907767988},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367415843588156,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:50.507163816+08:00","start-ts":457114712998150145,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114714243334149} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718267244555,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09e18413-4d6a-4b4c-a708-67eecbe0073b {"id":"09e18413-4d6a-4b4c-a708-67eecbe0073b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754270} /tidb/cdc/default/__cdc_meta__/capture/2cc294bc-3bc8-4688-a3c9-3cf756e2d677 {"id":"2cc294bc-3bc8-4688-a3c9-3cf756e2d677","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda07326f 09e18413-4d6a-4b4c-a708-67eecbe0073b /tidb/cdc/default/__cdc_meta__/owner/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/owner/223195ffda07326f 09e18413-4d6a-4b4c-a708-67eecbe0073b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10972440859670203073,"high":14428803174907767988},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367415843588156,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:50.507163816+08:00","start-ts":457114712998150145,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114714243334149} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718267244555,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09e18413-4d6a-4b4c-a708-67eecbe0073b {"id":"09e18413-4d6a-4b4c-a708-67eecbe0073b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754270} /tidb/cdc/default/__cdc_meta__/capture/2cc294bc-3bc8-4688-a3c9-3cf756e2d677 {"id":"2cc294bc-3bc8-4688-a3c9-3cf756e2d677","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754254} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda07326f 09e18413-4d6a-4b4c-a708-67eecbe0073b /tidb/cdc/default/__cdc_meta__/owner/223195ffda072fc2 2cc294bc-3bc8-4688-a3c9-3cf756e2d677 /tidb/cdc/default/__cdc_meta__/owner/223195ffda07326f 09e18413-4d6a-4b4c-a708-67eecbe0073b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10972440859670203073,"high":14428803174907767988},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367415843588156,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:50.507163816+08:00","start-ts":457114712998150145,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114714243334149} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718267244555,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.12438.out cli tso query --pd=http://127.0.0.1:2379 command: check_safepoint_equal http://127.0.0.1:2379 7489367088931747056 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d093c45-2392-4b3f-b0e1-ee283eb91c06 {"id":"3d093c45-2392-4b3f-b0e1-ee283eb91c06","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754270} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718527553551,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d093c45-2392-4b3f-b0e1-ee283eb91c06 {"id":"3d093c45-2392-4b3f-b0e1-ee283eb91c06","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754270} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718527553551,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d093c45-2392-4b3f-b0e1-ee283eb91c06 {"id":"3d093c45-2392-4b3f-b0e1-ee283eb91c06","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754270} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718527553551,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:11:13 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1616416166.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseE-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0dc7a6a4-5575-480b-bf0a-73657fdab427 {"id":"0dc7a6a4-5575-480b-bf0a-73657fdab427","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754271} /tidb/cdc/default/__cdc_meta__/capture/f5999f63-d209-462c-b50e-03c2d4b94130 {"id":"f5999f63-d209-462c-b50e-03c2d4b94130","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d971ba 0dc7a6a4-5575-480b-bf0a-73657fdab427 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d971ba 0dc7a6a4-5575-480b-bf0a-73657fdab427 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4922064178680103469,"high":15056557442061542273},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367362330872194,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:38.758872607+08:00","start-ts":457114709914288129,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711159472135} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714685046788,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0dc7a6a4-5575-480b-bf0a-73657fdab427 {"id":"0dc7a6a4-5575-480b-bf0a-73657fdab427","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754271} /tidb/cdc/default/__cdc_meta__/capture/f5999f63-d209-462c-b50e-03c2d4b94130 {"id":"f5999f63-d209-462c-b50e-03c2d4b94130","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d971ba 0dc7a6a4-5575-480b-bf0a-73657fdab427 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d971ba 0dc7a6a4-5575-480b-bf0a-73657fdab427 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4922064178680103469,"high":15056557442061542273},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367362330872194,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:38.758872607+08:00","start-ts":457114709914288129,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711159472135} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714685046788,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0dc7a6a4-5575-480b-bf0a-73657fdab427 {"id":"0dc7a6a4-5575-480b-bf0a-73657fdab427","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754271} /tidb/cdc/default/__cdc_meta__/capture/f5999f63-d209-462c-b50e-03c2d4b94130 {"id":"f5999f63-d209-462c-b50e-03c2d4b94130","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754244} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9d971ba 0dc7a6a4-5575-480b-bf0a-73657fdab427 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d96ce5 f5999f63-d209-462c-b50e-03c2d4b94130 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9d971ba 0dc7a6a4-5575-480b-bf0a-73657fdab427 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4922064178680103469,"high":15056557442061542273},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367362330872194,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:38.758872607+08:00","start-ts":457114709914288129,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711159472135} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114714685046788,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15924) - No such process wait process 15924 exit for 1-th time... process 15924 already exit [Fri Apr 4 16:11:11 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1609316095.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/226f2859-e361-4f6c-a4d1-2f9f98291ec3 {"id":"226f2859-e361-4f6c-a4d1-2f9f98291ec3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754245} /tidb/cdc/default/__cdc_meta__/capture/2eb632a8-fce1-48d1-8c2b-d87619a484cf {"id":"2eb632a8-fce1-48d1-8c2b-d87619a484cf","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754271} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e5907d 2eb632a8-fce1-48d1-8c2b-d87619a484cf /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e5907d 2eb632a8-fce1-48d1-8c2b-d87619a484cf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13206254903965006868,"high":5483312206236578955},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375875505646,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.417314802+08:00","start-ts":457114710610018305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711855202311} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715616968718,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/226f2859-e361-4f6c-a4d1-2f9f98291ec3 {"id":"226f2859-e361-4f6c-a4d1-2f9f98291ec3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754245} /tidb/cdc/default/__cdc_meta__/capture/2eb632a8-fce1-48d1-8c2b-d87619a484cf {"id":"2eb632a8-fce1-48d1-8c2b-d87619a484cf","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754271} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e5907d 2eb632a8-fce1-48d1-8c2b-d87619a484cf /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e5907d 2eb632a8-fce1-48d1-8c2b-d87619a484cf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13206254903965006868,"high":5483312206236578955},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375875505646,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.417314802+08:00","start-ts":457114710610018305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711855202311} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715616968718,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/226f2859-e361-4f6c-a4d1-2f9f98291ec3 {"id":"226f2859-e361-4f6c-a4d1-2f9f98291ec3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754245} /tidb/cdc/default/__cdc_meta__/capture/2eb632a8-fce1-48d1-8c2b-d87619a484cf {"id":"2eb632a8-fce1-48d1-8c2b-d87619a484cf","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754271} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e5907d 2eb632a8-fce1-48d1-8c2b-d87619a484cf /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e58bc2 226f2859-e361-4f6c-a4d1-2f9f98291ec3 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e5907d 2eb632a8-fce1-48d1-8c2b-d87619a484cf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13206254903965006868,"high":5483312206236578955},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375875505646,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.417314802+08:00","start-ts":457114710610018305,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711855202311} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715616968718,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x + tso='457114720179847169 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114720179847169 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:14 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1516115163.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (27635) - No such process wait process 27635 exit for 1-th time... process 27635 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (27712) - No such process wait process 27712 exit for 1-th time... process 27712 already exit [Fri Apr 4 16:11:13 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.2787927881.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114720393494529 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114720393494529 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:15 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1247312475.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_D Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:15 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d79f050-2107-49dd-83a1-2a5484d05590 {"id":"3d79f050-2107-49dd-83a1-2a5484d05590","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754272} /tidb/cdc/default/__cdc_meta__/capture/d61c8b00-ea01-4aad-9d93-349c5dfa7bcc {"id":"d61c8b00-ea01-4aad-9d93-349c5dfa7bcc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754269} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1058fd 3d79f050-2107-49dd-83a1-2a5484d05590 /tidb/cdc/default/__cdc_meta__/owner/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/__cdc_meta__/owner/223195ffda1058fd 3d79f050-2107-49dd-83a1-2a5484d05590 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114717846241291,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d79f050-2107-49dd-83a1-2a5484d05590 {"id":"3d79f050-2107-49dd-83a1-2a5484d05590","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754272} /tidb/cdc/default/__cdc_meta__/capture/d61c8b00-ea01-4aad-9d93-349c5dfa7bcc {"id":"d61c8b00-ea01-4aad-9d93-349c5dfa7bcc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754269} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1058fd 3d79f050-2107-49dd-83a1-2a5484d05590 /tidb/cdc/default/__cdc_meta__/owner/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/__cdc_meta__/owner/223195ffda1058fd 3d79f050-2107-49dd-83a1-2a5484d05590 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114717846241291,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d79f050-2107-49dd-83a1-2a5484d05590 {"id":"3d79f050-2107-49dd-83a1-2a5484d05590","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754272} /tidb/cdc/default/__cdc_meta__/capture/d61c8b00-ea01-4aad-9d93-349c5dfa7bcc {"id":"d61c8b00-ea01-4aad-9d93-349c5dfa7bcc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754269} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1056c4 22cdb486-c505-4001-8361-36481330a919 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1058fd 3d79f050-2107-49dd-83a1-2a5484d05590 /tidb/cdc/default/__cdc_meta__/owner/223195ffda105897 d61c8b00-ea01-4aad-9d93-349c5dfa7bcc /tidb/cdc/default/__cdc_meta__/owner/223195ffda1058fd 3d79f050-2107-49dd-83a1-2a5484d05590 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10685424696973760219,"high":2152855705924863395},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367424752522402,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:53.902309544+08:00","start-ts":457114713874759681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715133050884} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114717846241291,"progress":1}' + break + set +x check diff failed 1-th time, retry later command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 Changefeed remove successfully. ID: 1752807653410833484610169415565702060686 CheckpointTs: 457114717577019394 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 command: check_safepoint_forward http://127.0.0.1:2379 7489367088931747056 457114714418446340 457114717577019394 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCase-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=4 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:16 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d093c45-2392-4b3f-b0e1-ee283eb91c06 {"id":"3d093c45-2392-4b3f-b0e1-ee283eb91c06","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754270} /tidb/cdc/default/__cdc_meta__/capture/79ca4401-963b-4ce0-a004-491c80fd3785 {"id":"79ca4401-963b-4ce0-a004-491c80fd3785","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754273} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cde54 79ca4401-963b-4ce0-a004-491c80fd3785 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cde54 79ca4401-963b-4ce0-a004-491c80fd3785 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718527553551,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d093c45-2392-4b3f-b0e1-ee283eb91c06 {"id":"3d093c45-2392-4b3f-b0e1-ee283eb91c06","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754270} /tidb/cdc/default/__cdc_meta__/capture/79ca4401-963b-4ce0-a004-491c80fd3785 {"id":"79ca4401-963b-4ce0-a004-491c80fd3785","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754273} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cde54 79ca4401-963b-4ce0-a004-491c80fd3785 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cde54 79ca4401-963b-4ce0-a004-491c80fd3785 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718527553551,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d093c45-2392-4b3f-b0e1-ee283eb91c06 {"id":"3d093c45-2392-4b3f-b0e1-ee283eb91c06","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754270} /tidb/cdc/default/__cdc_meta__/capture/79ca4401-963b-4ce0-a004-491c80fd3785 {"id":"79ca4401-963b-4ce0-a004-491c80fd3785","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754273} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdcea 4509bda9-dde6-41f1-8dcf-77d34e671e4d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cde54 79ca4401-963b-4ce0-a004-491c80fd3785 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cde54 79ca4401-963b-4ce0-a004-491c80fd3785 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114718527553551,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseL-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:16 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13cc8639-689b-42e3-be8b-caadd9c8b329 {"id":"13cc8639-689b-42e3-be8b-caadd9c8b329","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754274} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715086651399,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13cc8639-689b-42e3-be8b-caadd9c8b329 {"id":"13cc8639-689b-42e3-be8b-caadd9c8b329","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754274} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715086651399,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13cc8639-689b-42e3-be8b-caadd9c8b329 {"id":"13cc8639-689b-42e3-be8b-caadd9c8b329","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754274} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715086651399,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:16 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.2794927951.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13863) - No such process wait process 13863 exit for 1-th time... process 13863 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (13930) - No such process wait process 13930 exit for 1-th time... process 13930 already exit [Fri Apr 4 16:11:15 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1406014062.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later run task successfully Changefeed remove successfully. ID: 151511097882765978997140042014189208459 CheckpointTs: 457114720460865541 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 command: check_safepoint_cleared http://127.0.0.1:2379 7489367088931747056 run task successfully check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:17 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64fa1488-95fb-4cbf-adfa-157bfcba92d9 {"id":"64fa1488-95fb-4cbf-adfa-157bfcba92d9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754274} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64fa1488-95fb-4cbf-adfa-157bfcba92d9 {"id":"64fa1488-95fb-4cbf-adfa-157bfcba92d9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754274} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64fa1488-95fb-4cbf-adfa-157bfcba92d9 {"id":"64fa1488-95fb-4cbf-adfa-157bfcba92d9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754274} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114720179847169 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.15221.out cli changefeed create --start-ts=457114720179847169 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367534224812083,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:17.898131488+08:00","start_ts":457114720179847169,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114720179847169,"checkpoint_ts":457114720179847169,"checkpoint_time":"2025-04-04 16:11:13.147","gid":{"low":8523928339396922107,"high":12761984788491403393}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367559394130666 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6ac6980002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:12646, start at 2025-04-04 16:11:17.287676852 +0800 CST m=+3.393775996 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:17.295 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:17.286 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:17.286 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367559394130666 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6ac6980002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:12646, start at 2025-04-04 16:11:17.287676852 +0800 CST m=+3.393775996 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:17.295 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:17.286 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:17.286 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367563956753628 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6ac7bc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-qqkbz--hsrzf, pid:12715, start at 2025-04-04 16:11:17.37203041 +0800 CST m=+3.367705662 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:17.381 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:17.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:17.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit log files: /tmp/tidb_cdc_test/gc_safepoint/stdout.log no DATA RACE found [Fri Apr 4 16:11:18 CST 2025] <<<<<< run test case gc_safepoint success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33ccfa89-5b53-4f63-914a-1b25702aa052 {"id":"33ccfa89-5b53-4f63-914a-1b25702aa052","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754275} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114719977209861,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33ccfa89-5b53-4f63-914a-1b25702aa052 {"id":"33ccfa89-5b53-4f63-914a-1b25702aa052","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754275} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114719977209861,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33ccfa89-5b53-4f63-914a-1b25702aa052 {"id":"33ccfa89-5b53-4f63-914a-1b25702aa052","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754275} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114719977209861,"progress":1}' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:18 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1412414126.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Starting Upstream TiDB... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_H/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:18 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12f821c2-7fe6-47eb-8132-4685e6bbdbd1 {"id":"12f821c2-7fe6-47eb-8132-4685e6bbdbd1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754275} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12f821c2-7fe6-47eb-8132-4685e6bbdbd1 {"id":"12f821c2-7fe6-47eb-8132-4685e6bbdbd1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754275} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12f821c2-7fe6-47eb-8132-4685e6bbdbd1 {"id":"12f821c2-7fe6-47eb-8132-4685e6bbdbd1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754275} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114720393494529 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.12535.out cli changefeed create --start-ts=457114720393494529 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367536119683961,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:18.745022759+08:00","start_ts":457114720393494529,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114720393494529,"checkpoint_ts":457114720393494529,"checkpoint_time":"2025-04-04 16:11:13.962","gid":{"low":9605402568531767544,"high":8153563322789178034}} PASS Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.7% of statements in github.com/pingcap/ticdc/... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16093) - No such process wait process 16093 exit for 1-th time... process 16093 already exit check diff failed 4-th time, retry later command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16169) - No such process wait process 16169 exit for 1-th time... process 16169 already exit [Fri Apr 4 16:11:20 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1626416266.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseG-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_G/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_G/stdout0-2.log /tmp/tidb_cdc_test/fail_over_ddl_G/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_G/stdout1-1.log /tmp/tidb_cdc_test/fail_over_ddl_G/stdout1-2.log no DATA RACE found [Fri Apr 4 16:11:20 CST 2025] <<<<<< run test case fail_over_ddl_G success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.13966.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:19 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13cc8639-689b-42e3-be8b-caadd9c8b329 {"id":"13cc8639-689b-42e3-be8b-caadd9c8b329","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754274} /tidb/cdc/default/__cdc_meta__/capture/f01a1cc1-a06d-42c9-88e4-5c45c644f92a {"id":"f01a1cc1-a06d-42c9-88e4-5c45c644f92a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754277} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e35621 f01a1cc1-a06d-42c9-88e4-5c45c644f92a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e35621 f01a1cc1-a06d-42c9-88e4-5c45c644f92a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715086651399,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13cc8639-689b-42e3-be8b-caadd9c8b329 {"id":"13cc8639-689b-42e3-be8b-caadd9c8b329","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754274} /tidb/cdc/default/__cdc_meta__/capture/f01a1cc1-a06d-42c9-88e4-5c45c644f92a {"id":"f01a1cc1-a06d-42c9-88e4-5c45c644f92a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754277} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e35621 f01a1cc1-a06d-42c9-88e4-5c45c644f92a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e35621 f01a1cc1-a06d-42c9-88e4-5c45c644f92a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715086651399,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13cc8639-689b-42e3-be8b-caadd9c8b329 {"id":"13cc8639-689b-42e3-be8b-caadd9c8b329","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754274} /tidb/cdc/default/__cdc_meta__/capture/f01a1cc1-a06d-42c9-88e4-5c45c644f92a {"id":"f01a1cc1-a06d-42c9-88e4-5c45c644f92a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754277} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e350ef 38675480-1c67-4771-89f7-fd32ef3245a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9e35621 f01a1cc1-a06d-42c9-88e4-5c45c644f92a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e355c0 13cc8639-689b-42e3-be8b-caadd9c8b329 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9e35621 f01a1cc1-a06d-42c9-88e4-5c45c644f92a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2544482256124478383,"high":12326019957456358587},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367375967016301,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:41.349883251+08:00","start-ts":457114710590881793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114711836327944} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114715086651399,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table fail_over_ddl_test.test2 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33ccfa89-5b53-4f63-914a-1b25702aa052 {"id":"33ccfa89-5b53-4f63-914a-1b25702aa052","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754275} /tidb/cdc/default/__cdc_meta__/capture/91881dff-936b-426a-a17d-d69fe379d29a {"id":"91881dff-936b-426a-a17d-d69fe379d29a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754278} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f55176 91881dff-936b-426a-a17d-d69fe379d29a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f55176 91881dff-936b-426a-a17d-d69fe379d29a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114719977209861,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33ccfa89-5b53-4f63-914a-1b25702aa052 {"id":"33ccfa89-5b53-4f63-914a-1b25702aa052","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754275} /tidb/cdc/default/__cdc_meta__/capture/91881dff-936b-426a-a17d-d69fe379d29a {"id":"91881dff-936b-426a-a17d-d69fe379d29a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754278} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f55176 91881dff-936b-426a-a17d-d69fe379d29a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f55176 91881dff-936b-426a-a17d-d69fe379d29a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114719977209861,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33ccfa89-5b53-4f63-914a-1b25702aa052 {"id":"33ccfa89-5b53-4f63-914a-1b25702aa052","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754275} /tidb/cdc/default/__cdc_meta__/capture/91881dff-936b-426a-a17d-d69fe379d29a {"id":"91881dff-936b-426a-a17d-d69fe379d29a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754278} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f54cc3 3d697161-5c5c-4aa7-8212-8eaf9fb3c2f8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffd9f55176 91881dff-936b-426a-a17d-d69fe379d29a /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f55115 33ccfa89-5b53-4f63-914a-1b25702aa052 /tidb/cdc/default/__cdc_meta__/owner/223195ffd9f55176 91881dff-936b-426a-a17d-d69fe379d29a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":955649239624675763,"high":18012802736310228379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367390112237633,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:46.902925948+08:00","start-ts":457114712047353857,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114713292537864} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114719977209861,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 5-th time, retry later check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15166) - No such process wait process 15166 exit for 1-th time... process 15166 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:11:21 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1530315305.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_E Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x + tso='457114722195210241 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114722195210241 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.1399613998.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (12478) - No such process wait process 12478 exit for 1-th time... process 12478 already exit [Fri Apr 4 16:11:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1261812620.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_L Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367584068670880 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6b11080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:14914, start at 2025-04-04 16:11:22.084323118 +0800 CST m=+3.402545303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:22.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:22.150 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:22.150 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367584068670880 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6b11080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:14914, start at 2025-04-04 16:11:22.084323118 +0800 CST m=+3.402545303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:22.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:22.150 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:22.150 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367590292783755 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6b166c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-7m7s5--lfsfz, pid:15009, start at 2025-04-04 16:11:22.401332041 +0800 CST m=+3.534472010 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:22.449 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:22.446 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:22.446 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed22148c-f6ed-4446-96ec-44a7e05492a3 {"id":"ed22148c-f6ed-4446-96ec-44a7e05492a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754280} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721227636748,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed22148c-f6ed-4446-96ec-44a7e05492a3 {"id":"ed22148c-f6ed-4446-96ec-44a7e05492a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754280} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721227636748,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed22148c-f6ed-4446-96ec-44a7e05492a3 {"id":"ed22148c-f6ed-4446-96ec-44a7e05492a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754280} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721227636748,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":2}' + echo '{"count":2}' {"count":2} ++ echo '{"count":2}' ++ jq -r .count + value=2 + '[' -1 == -1 ']' + '[' 2 '!=' null ']' + exit 0 [Fri Apr 4 16:11:23 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1634016342.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_D/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 5500 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15888) - No such process wait process 15888 exit for 1-th time... process 15888 already exit [Fri Apr 4 16:11:21 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1607316075.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseC-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_C/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_C/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_C/stdout1-1.log /tmp/tidb_cdc_test/fail_over_ddl_C/stdout1-2.log no DATA RACE found [Fri Apr 4 16:11:24 CST 2025] <<<<<< run test case fail_over_ddl_C success! >>>>>> check diff failed 2-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_I Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4108600-ea15-4ade-a23f-723deb24af56 {"id":"a4108600-ea15-4ade-a23f-723deb24af56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/capture/b276a371-4b60-4fbd-a770-4731c712dfeb {"id":"b276a371-4b60-4fbd-a770-4731c712dfeb","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1755f1 b276a371-4b60-4fbd-a770-4731c712dfeb /tidb/cdc/default/__cdc_meta__/owner/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1755f1 b276a371-4b60-4fbd-a770-4731c712dfeb /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9389614194233293602,"high":17585974546411942538},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367431094912519,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:54.607731103+08:00","start-ts":457114714069008386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715313930248} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114719101911053,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4108600-ea15-4ade-a23f-723deb24af56 {"id":"a4108600-ea15-4ade-a23f-723deb24af56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/capture/b276a371-4b60-4fbd-a770-4731c712dfeb {"id":"b276a371-4b60-4fbd-a770-4731c712dfeb","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1755f1 b276a371-4b60-4fbd-a770-4731c712dfeb /tidb/cdc/default/__cdc_meta__/owner/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1755f1 b276a371-4b60-4fbd-a770-4731c712dfeb /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9389614194233293602,"high":17585974546411942538},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367431094912519,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:54.607731103+08:00","start-ts":457114714069008386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715313930248} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114719101911053,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4108600-ea15-4ade-a23f-723deb24af56 {"id":"a4108600-ea15-4ade-a23f-723deb24af56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754259} /tidb/cdc/default/__cdc_meta__/capture/b276a371-4b60-4fbd-a770-4731c712dfeb {"id":"b276a371-4b60-4fbd-a770-4731c712dfeb","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda1755f1 b276a371-4b60-4fbd-a770-4731c712dfeb /tidb/cdc/default/__cdc_meta__/owner/223195ffda1751c3 a4108600-ea15-4ade-a23f-723deb24af56 /tidb/cdc/default/__cdc_meta__/owner/223195ffda1755f1 b276a371-4b60-4fbd-a770-4731c712dfeb /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9389614194233293602,"high":17585974546411942538},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367431094912519,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:10:54.607731103+08:00","start-ts":457114714069008386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114715313930248} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114719101911053,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c08f1cbd-8327-4e92-83a5-2791a69ea2e4 {"id":"c08f1cbd-8327-4e92-83a5-2791a69ea2e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8523928339396922107,"high":12761984788491403393},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367534224812083,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:17.898131488+08:00","start-ts":457114720179847169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721425293314} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721792032807,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c08f1cbd-8327-4e92-83a5-2791a69ea2e4 {"id":"c08f1cbd-8327-4e92-83a5-2791a69ea2e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8523928339396922107,"high":12761984788491403393},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367534224812083,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:17.898131488+08:00","start-ts":457114720179847169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721425293314} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721792032807,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c08f1cbd-8327-4e92-83a5-2791a69ea2e4 {"id":"c08f1cbd-8327-4e92-83a5-2791a69ea2e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8523928339396922107,"high":12761984788491403393},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367534224812083,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:17.898131488+08:00","start-ts":457114720179847169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721425293314} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114721792032807,"progress":1}' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:25 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1538315385.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.cli.16341.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:25 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/daaefcee-ded3-4eb9-b60e-8d9fa698abd2 {"id":"daaefcee-ded3-4eb9-b60e-8d9fa698abd2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/daaefcee-ded3-4eb9-b60e-8d9fa698abd2 {"id":"daaefcee-ded3-4eb9-b60e-8d9fa698abd2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/daaefcee-ded3-4eb9-b60e-8d9fa698abd2 {"id":"daaefcee-ded3-4eb9-b60e-8d9fa698abd2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114722195210241 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.14071.out cli changefeed create --start-ts=457114722195210241 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367559394130666,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:25.597975129+08:00","start_ts":457114722195210241,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114722195210241,"checkpoint_ts":457114722195210241,"checkpoint_time":"2025-04-04 16:11:20.835","gid":{"low":12341241844399578707,"high":7306502954943539847}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ff5c030-4c8e-45f6-86ba-98afa63d7566 {"id":"6ff5c030-4c8e-45f6-86ba-98afa63d7566","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754283} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9605402568531767544,"high":8153563322789178034},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367536119683961,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:18.745022759+08:00","start-ts":457114720393494529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721638678537} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114722018787367,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ff5c030-4c8e-45f6-86ba-98afa63d7566 {"id":"6ff5c030-4c8e-45f6-86ba-98afa63d7566","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754283} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9605402568531767544,"high":8153563322789178034},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367536119683961,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:18.745022759+08:00","start-ts":457114720393494529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721638678537} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114722018787367,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ff5c030-4c8e-45f6-86ba-98afa63d7566 {"id":"6ff5c030-4c8e-45f6-86ba-98afa63d7566","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754283} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9605402568531767544,"high":8153563322789178034},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367536119683961,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:18.745022759+08:00","start-ts":457114720393494529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721638678537} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114722018787367,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:25 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=1*sleep(30000)->return(false)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1270612708.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 Verifying downstream PD is started... check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseK-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseM-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:26 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/828b9374-8cae-4e26-969b-12197b977e9c {"id":"828b9374-8cae-4e26-969b-12197b977e9c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754283} /tidb/cdc/default/__cdc_meta__/capture/ed22148c-f6ed-4446-96ec-44a7e05492a3 {"id":"ed22148c-f6ed-4446-96ec-44a7e05492a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754280} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdfed 828b9374-8cae-4e26-969b-12197b977e9c /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdfed 828b9374-8cae-4e26-969b-12197b977e9c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721227636748,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/828b9374-8cae-4e26-969b-12197b977e9c {"id":"828b9374-8cae-4e26-969b-12197b977e9c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754283} /tidb/cdc/default/__cdc_meta__/capture/ed22148c-f6ed-4446-96ec-44a7e05492a3 {"id":"ed22148c-f6ed-4446-96ec-44a7e05492a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754280} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdfed 828b9374-8cae-4e26-969b-12197b977e9c /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdfed 828b9374-8cae-4e26-969b-12197b977e9c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721227636748,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/828b9374-8cae-4e26-969b-12197b977e9c {"id":"828b9374-8cae-4e26-969b-12197b977e9c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754283} /tidb/cdc/default/__cdc_meta__/capture/ed22148c-f6ed-4446-96ec-44a7e05492a3 {"id":"ed22148c-f6ed-4446-96ec-44a7e05492a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754280} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cddf1 3d093c45-2392-4b3f-b0e1-ee283eb91c06 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda3cdfed 828b9374-8cae-4e26-969b-12197b977e9c /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdf89 ed22148c-f6ed-4446-96ec-44a7e05492a3 /tidb/cdc/default/__cdc_meta__/owner/223195ffda3cdfed 828b9374-8cae-4e26-969b-12197b977e9c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":17458474664049934162,"high":8672975050330466229},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367466750404009,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:04.873575671+08:00","start-ts":457114716758081537,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114718003265544} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721227636748,"progress":1}' + grep -q 'etcd info' + break + set +x + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + set +x + tso='457114723457433601 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114723457433601 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:27 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1637716379.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 2-th time, retry later command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 4-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... check diff failed 1-th time, retry later check diff failed 1-th time, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92cdd691-4aea-4a30-ae41-ee0b8aab4632 {"id":"92cdd691-4aea-4a30-ae41-ee0b8aab4632","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754285} /tidb/cdc/default/__cdc_meta__/capture/c08f1cbd-8327-4e92-83a5-2791a69ea2e4 {"id":"c08f1cbd-8327-4e92-83a5-2791a69ea2e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8523928339396922107,"high":12761984788491403393},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367534224812083,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:17.898131488+08:00","start-ts":457114720179847169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721425293314} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721805402119,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92cdd691-4aea-4a30-ae41-ee0b8aab4632 {"id":"92cdd691-4aea-4a30-ae41-ee0b8aab4632","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754285} /tidb/cdc/default/__cdc_meta__/capture/c08f1cbd-8327-4e92-83a5-2791a69ea2e4 {"id":"c08f1cbd-8327-4e92-83a5-2791a69ea2e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8523928339396922107,"high":12761984788491403393},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367534224812083,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:17.898131488+08:00","start-ts":457114720179847169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721425293314} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721805402119,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92cdd691-4aea-4a30-ae41-ee0b8aab4632 {"id":"92cdd691-4aea-4a30-ae41-ee0b8aab4632","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754285} /tidb/cdc/default/__cdc_meta__/capture/c08f1cbd-8327-4e92-83a5-2791a69ea2e4 {"id":"c08f1cbd-8327-4e92-83a5-2791a69ea2e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754282} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780be5 64fa1488-95fb-4cbf-adfa-157bfcba92d9 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780cc5 c08f1cbd-8327-4e92-83a5-2791a69ea2e4 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8523928339396922107,"high":12761984788491403393},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367534224812083,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:17.898131488+08:00","start-ts":457114720179847169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721425293314} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114721805402119,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.15445.out cli capture list [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ff5c030-4c8e-45f6-86ba-98afa63d7566 {"id":"6ff5c030-4c8e-45f6-86ba-98afa63d7566","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754283} /tidb/cdc/default/__cdc_meta__/capture/a97d8c28-1fef-45f3-9e80-4fee4162f40f {"id":"a97d8c28-1fef-45f3-9e80-4fee4162f40f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754286} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/__cdc_meta__/owner/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9605402568531767544,"high":8153563322789178034},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367536119683961,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:18.745022759+08:00","start-ts":457114720393494529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721638678537} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114722018787367,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ff5c030-4c8e-45f6-86ba-98afa63d7566 {"id":"6ff5c030-4c8e-45f6-86ba-98afa63d7566","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754283} /tidb/cdc/default/__cdc_meta__/capture/a97d8c28-1fef-45f3-9e80-4fee4162f40f {"id":"a97d8c28-1fef-45f3-9e80-4fee4162f40f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754286} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/__cdc_meta__/owner/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9605402568531767544,"high":8153563322789178034},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367536119683961,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:18.745022759+08:00","start-ts":457114720393494529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721638678537} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114722018787367,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ff5c030-4c8e-45f6-86ba-98afa63d7566 {"id":"6ff5c030-4c8e-45f6-86ba-98afa63d7566","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754283} /tidb/cdc/default/__cdc_meta__/capture/a97d8c28-1fef-45f3-9e80-4fee4162f40f {"id":"a97d8c28-1fef-45f3-9e80-4fee4162f40f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754286} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a8e5 12f821c2-7fe6-47eb-8132-4685e6bbdbd1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/__cdc_meta__/owner/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9605402568531767544,"high":8153563322789178034},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367536119683961,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:18.745022759+08:00","start-ts":457114720393494529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721638678537} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114722018787367,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.12784.out cli capture list check diff failed 3-th time, retry later table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (14001) - No such process wait process 14001 exit for 1-th time... check diff failed 2-th time, retry later + set +x + ans='[ { "id": "c08f1cbd-8327-4e92-83a5-2791a69ea2e4", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "92cdd691-4aea-4a30-ae41-ee0b8aab4632", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"c08f1cbd-8327-4e92-83a5-2791a69ea2e4",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"92cdd691-4aea-4a30-ae41-ee0b8aab4632",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=92cdd691-4aea-4a30-ae41-ee0b8aab4632 + '[' -z 92cdd691-4aea-4a30-ae41-ee0b8aab4632 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 92cdd691-4aea-4a30-ae41-ee0b8aab4632 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.15487.out cli changefeed move-table -c test -t 114 -d 92cdd691-4aea-4a30-ae41-ee0b8aab4632 process 14001 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:11:29 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.1413614138.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:30 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1fd047b-1666-4449-a32b-4a53e5a39a7d {"id":"c1fd047b-1666-4449-a32b-4a53e5a39a7d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754287} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1fd047b-1666-4449-a32b-4a53e5a39a7d {"id":"c1fd047b-1666-4449-a32b-4a53e5a39a7d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754287} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1fd047b-1666-4449-a32b-4a53e5a39a7d {"id":"c1fd047b-1666-4449-a32b-4a53e5a39a7d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754287} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114723457433601 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.cli.16441.out cli changefeed create --start-ts=457114723457433601 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367584068670880,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:30.433968958+08:00","start_ts":457114723457433601,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114723457433601,"checkpoint_ts":457114723457433601,"checkpoint_time":"2025-04-04 16:11:25.650","gid":{"low":16233957215137547470,"high":603714472361012659}} PASS [Pipeline] // cache check diff failed 2-th time, retry later [Pipeline] } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x + ans='[ { "id": "a97d8c28-1fef-45f3-9e80-4fee4162f40f", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" }, { "id": "6ff5c030-4c8e-45f6-86ba-98afa63d7566", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"a97d8c28-1fef-45f3-9e80-4fee4162f40f",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '},' '{' '"id":' '"6ff5c030-4c8e-45f6-86ba-98afa63d7566",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=a97d8c28-1fef-45f3-9e80-4fee4162f40f + '[' -z a97d8c28-1fef-45f3-9e80-4fee4162f40f ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d a97d8c28-1fef-45f3-9e80-4fee4162f40f + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.12815.out cli changefeed move-table -c test -t 114 -d a97d8c28-1fef-45f3-9e80-4fee4162f40f [Pipeline] } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later check diff failed 4-th time, retry later check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367615344892608 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6b9cd8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:17400, start at 2025-04-04 16:11:31.011791913 +0800 CST m=+3.085655116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:31.019 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:30.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:30.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367615344892608 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6b9cd8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:17400, start at 2025-04-04 16:11:31.011791913 +0800 CST m=+3.085655116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:31.019 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:30.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:30.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367625135293617 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6ba1480005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-0xql0--nnp7l, pid:17479, start at 2025-04-04 16:11:31.290323824 +0800 CST m=+3.203332985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:31.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:31.298 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:31.298 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_E/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseO-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseA-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 {"id":"b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754290} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12341241844399578707,"high":7306502954943539847},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367559394130666,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:25.597975129+08:00","start-ts":457114722195210241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114723440656389} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114723807657999,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 {"id":"b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754290} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12341241844399578707,"high":7306502954943539847},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367559394130666,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:25.597975129+08:00","start-ts":457114722195210241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114723440656389} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114723807657999,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 {"id":"b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754290} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12341241844399578707,"high":7306502954943539847},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367559394130666,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:25.597975129+08:00","start-ts":457114722195210241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114723440656389} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114723807657999,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:32 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.1421714219.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367628688494762 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6bad580005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:14609, start at 2025-04-04 16:11:32.05795305 +0800 CST m=+3.061174419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:32.064 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:32.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:32.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367628688494762 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6bad580005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:14609, start at 2025-04-04 16:11:32.05795305 +0800 CST m=+3.061174419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:32.064 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:32.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:32.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367619803607227 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6bb1f4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:14667, start at 2025-04-04 16:11:32.375774246 +0800 CST m=+3.223786049 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:32.385 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:32.349 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:32.349 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseJ-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [Pipeline] // cache [Pipeline] } [2025/04/04 16:11:33.955 +08:00] [DEBUG] [pd.rs:582] ["update latency statistics for raftstore-v1"] [tick=425] [thread_id=88] [2025/04/04 16:11:33.955 +08:00] [DEBUG] [pool.rs:146] ["scheduling task inspect raftstore latency: RaftDisk"] [thread_id=88] [2025/04/04 16:11:33.955 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="inspect raftstore latency: RaftDisk"] [thread_id=30] [2025/04/04 16:11:33.955 +08:00] [DEBUG] [pool.rs:146] ["scheduling task compute slow score: id 426, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(24.734ยตs), store_process_duration: Some(8.536ยตs), store_write_duration: Some(16.553ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=80] [2025/04/04 16:11:33.955 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="compute slow score: id 426, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(24.734ยตs), store_process_duration: Some(8.536ยตs), store_write_duration: Some(16.553ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=30] [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16382) - No such process wait process 16382 exit for 1-th time... process 16382 already exit [Fri Apr 4 16:11:34 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1650516507.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // container [Pipeline] } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_K Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_M Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully [Pipeline] // podTemplate VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367627689970784 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6bc4200004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:14531, start at 2025-04-04 16:11:33.517596821 +0800 CST m=+3.150944891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:33.525 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:33.512 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:33.512 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367627689970784 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6bc4200004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:14531, start at 2025-04-04 16:11:33.517596821 +0800 CST m=+3.150944891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:33.525 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:33.512 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:33.512 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367639049943628 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6bc7100004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:14609, start at 2025-04-04 16:11:33.704910383 +0800 CST m=+3.173164102 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:33.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:33.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:33.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.18697.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } [Pipeline] // withEnv [Pipeline] } /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15308) - No such process wait process 15308 exit for 1-th time... process 15308 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 18200 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.16014.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457114725881741313 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114725881741313 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:36 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1873518737.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a52efcc0-736d-4773-b817-f79548b56849 {"id":"a52efcc0-736d-4773-b817-f79548b56849","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754293} /tidb/cdc/default/__cdc_meta__/capture/b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 {"id":"b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754290} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12341241844399578707,"high":7306502954943539847},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367559394130666,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:25.597975129+08:00","start-ts":457114722195210241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114723440656389} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114723820765204,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a52efcc0-736d-4773-b817-f79548b56849 {"id":"a52efcc0-736d-4773-b817-f79548b56849","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754293} /tidb/cdc/default/__cdc_meta__/capture/b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 {"id":"b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754290} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12341241844399578707,"high":7306502954943539847},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367559394130666,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:25.597975129+08:00","start-ts":457114722195210241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114723440656389} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114723820765204,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a52efcc0-736d-4773-b817-f79548b56849 {"id":"a52efcc0-736d-4773-b817-f79548b56849","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754293} /tidb/cdc/default/__cdc_meta__/capture/b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 {"id":"b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754290} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c51e9 daaefcee-ded3-4eb9-b60e-8d9fa698abd2 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c52c6 b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12341241844399578707,"high":7306502954943539847},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367559394130666,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:25.597975129+08:00","start-ts":457114722195210241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114723440656389} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114723820765204,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.14293.out cli capture list chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + tso='457114726158303233 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114726158303233 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:37 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1605116053.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.15959.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 check diff failed 2-th time, retry later + set +x + ans='[ { "id": "b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "a52efcc0-736d-4773-b817-f79548b56849", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"b5e71a8e-cfdd-4527-bed6-9d7fa56a13d1",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"a52efcc0-736d-4773-b817-f79548b56849",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=a52efcc0-736d-4773-b817-f79548b56849 + '[' -z a52efcc0-736d-4773-b817-f79548b56849 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d a52efcc0-736d-4773-b817-f79548b56849 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.cli.14325.out cli changefeed move-table -c test -t 114 -d a52efcc0-736d-4773-b817-f79548b56849 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e4e06d2-e41a-4030-b2bc-a0dd86e92239 {"id":"5e4e06d2-e41a-4030-b2bc-a0dd86e92239","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754294} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725108940813,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e4e06d2-e41a-4030-b2bc-a0dd86e92239 {"id":"5e4e06d2-e41a-4030-b2bc-a0dd86e92239","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754294} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725108940813,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e4e06d2-e41a-4030-b2bc-a0dd86e92239 {"id":"5e4e06d2-e41a-4030-b2bc-a0dd86e92239","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754294} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725108940813,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:11:37 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1657716579.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + tso='457114726514819073 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114726514819073 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:38 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1599916001.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 91000 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:39 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cec40b0f-ae99-48e2-ad80-002a06660604 {"id":"cec40b0f-ae99-48e2-ad80-002a06660604","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754296} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cec40b0f-ae99-48e2-ad80-002a06660604 {"id":"cec40b0f-ae99-48e2-ad80-002a06660604","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754296} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cec40b0f-ae99-48e2-ad80-002a06660604 {"id":"cec40b0f-ae99-48e2-ad80-002a06660604","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754296} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114725881741313 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.18796.out cli changefeed create --start-ts=457114725881741313 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 Create changefeed successfully! ID: test Info: {"upstream_id":7489367615344892608,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:39.674226645+08:00","start_ts":457114725881741313,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114725881741313,"checkpoint_ts":457114725881741313,"checkpoint_time":"2025-04-04 16:11:34.898","gid":{"low":5136872072753862960,"high":17034506343219206841}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseF-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:40 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87e48756-869f-494a-9e2a-9ce99a437fba {"id":"87e48756-869f-494a-9e2a-9ce99a437fba","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754297} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba /tidb/cdc/default/__cdc_meta__/owner/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87e48756-869f-494a-9e2a-9ce99a437fba {"id":"87e48756-869f-494a-9e2a-9ce99a437fba","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754297} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba /tidb/cdc/default/__cdc_meta__/owner/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87e48756-869f-494a-9e2a-9ce99a437fba {"id":"87e48756-869f-494a-9e2a-9ce99a437fba","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754297} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba /tidb/cdc/default/__cdc_meta__/owner/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114726158303233 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.16112.out cli changefeed create --start-ts=457114726158303233 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367628688494762,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:40.705455139+08:00","start_ts":457114726158303233,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114726158303233,"checkpoint_ts":457114726158303233,"checkpoint_time":"2025-04-04 16:11:35.953","gid":{"low":15512654085710704676,"high":14444012998446361230}} PASS chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_O Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15b9f882-7a25-4c31-87c9-1400f456716b {"id":"15b9f882-7a25-4c31-87c9-1400f456716b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754297} /tidb/cdc/default/__cdc_meta__/capture/5e4e06d2-e41a-4030-b2bc-a0dd86e92239 {"id":"5e4e06d2-e41a-4030-b2bc-a0dd86e92239","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754294} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e728 15b9f882-7a25-4c31-87c9-1400f456716b /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e728 15b9f882-7a25-4c31-87c9-1400f456716b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725108940813,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15b9f882-7a25-4c31-87c9-1400f456716b {"id":"15b9f882-7a25-4c31-87c9-1400f456716b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754297} /tidb/cdc/default/__cdc_meta__/capture/5e4e06d2-e41a-4030-b2bc-a0dd86e92239 {"id":"5e4e06d2-e41a-4030-b2bc-a0dd86e92239","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754294} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e728 15b9f882-7a25-4c31-87c9-1400f456716b /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e728 15b9f882-7a25-4c31-87c9-1400f456716b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725108940813,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15b9f882-7a25-4c31-87c9-1400f456716b {"id":"15b9f882-7a25-4c31-87c9-1400f456716b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754297} /tidb/cdc/default/__cdc_meta__/capture/5e4e06d2-e41a-4030-b2bc-a0dd86e92239 {"id":"5e4e06d2-e41a-4030-b2bc-a0dd86e92239","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754294} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e5e3 c1fd047b-1666-4449-a32b-4a53e5a39a7d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e728 15b9f882-7a25-4c31-87c9-1400f456716b /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8e728 15b9f882-7a25-4c31-87c9-1400f456716b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725108940813,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":2}' + echo '{"count":2}' {"count":2} ++ echo '{"count":2}' ++ jq -r .count + value=2 + '[' -1 == -1 ']' + '[' 2 '!=' null ']' + exit 0 [Fri Apr 4 16:11:41 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1556915571.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_A Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:41 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d32e3b7a-2111-4b1e-a199-7f27b0b18e65 {"id":"d32e3b7a-2111-4b1e-a199-7f27b0b18e65","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754299} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d32e3b7a-2111-4b1e-a199-7f27b0b18e65 {"id":"d32e3b7a-2111-4b1e-a199-7f27b0b18e65","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754299} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d32e3b7a-2111-4b1e-a199-7f27b0b18e65 {"id":"d32e3b7a-2111-4b1e-a199-7f27b0b18e65","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754299} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114726514819073 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.16055.out cli changefeed create --start-ts=457114726514819073 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367627689970784,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:42.068524892+08:00","start_ts":457114726514819073,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114726514819073,"checkpoint_ts":457114726514819073,"checkpoint_time":"2025-04-04 16:11:37.313","gid":{"low":10830979984427073096,"high":7852405283103412372}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Command succeeded on attempt 1 run task successfully command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_J Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later table fail_over_ddl_test.test2 not exists for 1-th check, retry later table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18740) - No such process wait process 18740 exit for 1-th time... process 18740 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:11:43 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockorWaitBeforeWrite=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1886918871.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367674212210317 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6c5b94000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:29006, start at 2025-04-04 16:11:43.228675716 +0800 CST m=+3.214024069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:43.290 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:43.256 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:43.256 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367674212210317 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6c5b94000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:29006, start at 2025-04-04 16:11:43.228675716 +0800 CST m=+3.214024069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:43.290 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:43.256 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:43.256 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367670690562932 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6c5ea4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:29077, start at 2025-04-04 16:11:43.416451032 +0800 CST m=+3.288132131 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:43.424 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:43.482 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:43.482 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:44 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92cdd691-4aea-4a30-ae41-ee0b8aab4632 {"id":"92cdd691-4aea-4a30-ae41-ee0b8aab4632","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754285} /tidb/cdc/default/__cdc_meta__/capture/fb7bda4a-2794-4f94-b520-f23bb90375ff {"id":"fb7bda4a-2794-4f94-b520-f23bb90375ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754301} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780fcc fb7bda4a-2794-4f94-b520-f23bb90375ff /tidb/cdc/default/__cdc_meta__/owner/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780fcc fb7bda4a-2794-4f94-b520-f23bb90375ff /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8523928339396922107,"high":12761984788491403393},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367534224812083,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:17.898131488+08:00","start-ts":457114720179847169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721425293314} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725213011971,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92cdd691-4aea-4a30-ae41-ee0b8aab4632 {"id":"92cdd691-4aea-4a30-ae41-ee0b8aab4632","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754285} /tidb/cdc/default/__cdc_meta__/capture/fb7bda4a-2794-4f94-b520-f23bb90375ff {"id":"fb7bda4a-2794-4f94-b520-f23bb90375ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754301} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780fcc fb7bda4a-2794-4f94-b520-f23bb90375ff /tidb/cdc/default/__cdc_meta__/owner/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780fcc fb7bda4a-2794-4f94-b520-f23bb90375ff /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8523928339396922107,"high":12761984788491403393},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367534224812083,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:17.898131488+08:00","start-ts":457114720179847169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721425293314} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725213011971,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92cdd691-4aea-4a30-ae41-ee0b8aab4632 {"id":"92cdd691-4aea-4a30-ae41-ee0b8aab4632","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754285} /tidb/cdc/default/__cdc_meta__/capture/fb7bda4a-2794-4f94-b520-f23bb90375ff {"id":"fb7bda4a-2794-4f94-b520-f23bb90375ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754301} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda780fcc fb7bda4a-2794-4f94-b520-f23bb90375ff /tidb/cdc/default/__cdc_meta__/owner/223195ffda780d28 92cdd691-4aea-4a30-ae41-ee0b8aab4632 /tidb/cdc/default/__cdc_meta__/owner/223195ffda780fcc fb7bda4a-2794-4f94-b520-f23bb90375ff /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8523928339396922107,"high":12761984788491403393},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367534224812083,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:17.898131488+08:00","start-ts":457114720179847169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721425293314} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725213011971,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (14141) - No such process wait process 14141 exit for 1-th time... process 14141 already exit Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table fail_over_ddl_test.test2 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16004) - No such process wait process 16004 exit for 1-th time... process 16004 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:11:46 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1614416146.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16056) - No such process wait process 16056 exit for 1-th time... process 16056 already exit [Fri Apr 4 16:11:46 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1618516187.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... check diff failed 2-th time, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.30295.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 {"id":"f0eb9874-49dc-4e56-9eae-d08f3bf53bf0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754304} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5136872072753862960,"high":17034506343219206841},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367615344892608,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:39.674226645+08:00","start-ts":457114725881741313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727126925319} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727481081869,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 {"id":"f0eb9874-49dc-4e56-9eae-d08f3bf53bf0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754304} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5136872072753862960,"high":17034506343219206841},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367615344892608,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:39.674226645+08:00","start-ts":457114725881741313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727126925319} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727481081869,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 {"id":"f0eb9874-49dc-4e56-9eae-d08f3bf53bf0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754304} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5136872072753862960,"high":17034506343219206841},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367615344892608,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:39.674226645+08:00","start-ts":457114725881741313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727126925319} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727481081869,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:46 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1894718949.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_F Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x + tso='457114729042673666 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114729042673666 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:48 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.3033130333.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseB-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c82f854f-1294-43d2-b752-790c9c30ea54 {"id":"c82f854f-1294-43d2-b752-790c9c30ea54","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10830979984427073096,"high":7852405283103412372},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367627689970784,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:42.068524892+08:00","start-ts":457114726514819073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727760003077} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114728139849756,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c82f854f-1294-43d2-b752-790c9c30ea54 {"id":"c82f854f-1294-43d2-b752-790c9c30ea54","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10830979984427073096,"high":7852405283103412372},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367627689970784,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:42.068524892+08:00","start-ts":457114726514819073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727760003077} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114728139849756,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c82f854f-1294-43d2-b752-790c9c30ea54 {"id":"c82f854f-1294-43d2-b752-790c9c30ea54","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10830979984427073096,"high":7852405283103412372},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367627689970784,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:42.068524892+08:00","start-ts":457114726514819073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727760003077} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114728139849756,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:49 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1621716219.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3087 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c857d8f-b9db-448f-a282-2480fc8ef793 {"id":"0c857d8f-b9db-448f-a282-2480fc8ef793","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15512654085710704676,"high":14444012998446361230},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367628688494762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:40.705455139+08:00","start-ts":457114726158303233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727403487234} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727901560858,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c857d8f-b9db-448f-a282-2480fc8ef793 {"id":"0c857d8f-b9db-448f-a282-2480fc8ef793","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15512654085710704676,"high":14444012998446361230},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367628688494762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:40.705455139+08:00","start-ts":457114726158303233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727403487234} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727901560858,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c857d8f-b9db-448f-a282-2480fc8ef793 {"id":"0c857d8f-b9db-448f-a282-2480fc8ef793","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15512654085710704676,"high":14444012998446361230},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367628688494762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:40.705455139+08:00","start-ts":457114726158303233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727403487234} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727901560858,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:49 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1626216264.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040778c9-0001-42ce-bc0c-36cf3cac6f8e {"id":"040778c9-0001-42ce-bc0c-36cf3cac6f8e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754307} /tidb/cdc/default/__cdc_meta__/capture/f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 {"id":"f0eb9874-49dc-4e56-9eae-d08f3bf53bf0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754304} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/__cdc_meta__/owner/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5136872072753862960,"high":17034506343219206841},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367615344892608,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:39.674226645+08:00","start-ts":457114725881741313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727126925319} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727520141313,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040778c9-0001-42ce-bc0c-36cf3cac6f8e {"id":"040778c9-0001-42ce-bc0c-36cf3cac6f8e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754307} /tidb/cdc/default/__cdc_meta__/capture/f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 {"id":"f0eb9874-49dc-4e56-9eae-d08f3bf53bf0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754304} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/__cdc_meta__/owner/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5136872072753862960,"high":17034506343219206841},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367615344892608,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:39.674226645+08:00","start-ts":457114725881741313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727126925319} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727520141313,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040778c9-0001-42ce-bc0c-36cf3cac6f8e {"id":"040778c9-0001-42ce-bc0c-36cf3cac6f8e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754307} /tidb/cdc/default/__cdc_meta__/capture/f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 {"id":"f0eb9874-49dc-4e56-9eae-d08f3bf53bf0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754304} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac30fed cec40b0f-ae99-48e2-ad80-002a06660604 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/__cdc_meta__/owner/223195ffdac310cc f0eb9874-49dc-4e56-9eae-d08f3bf53bf0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5136872072753862960,"high":17034506343219206841},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367615344892608,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:39.674226645+08:00","start-ts":457114725881741313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727126925319} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727520141313,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.19017.out cli capture list TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367703208224136 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6cbb800002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:14667, start at 2025-04-04 16:11:49.345555797 +0800 CST m=+3.370013635 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:49.353 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:49.344 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:49.344 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367703208224136 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6cbb800002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:14667, start at 2025-04-04 16:11:49.345555797 +0800 CST m=+3.370013635 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:49.353 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:49.344 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:49.344 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367701531462498 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6cbea8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:14744, start at 2025-04-04 16:11:49.585362336 +0800 CST m=+3.439660828 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:49.593 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:49.596 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:49.596 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367707515850467 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6cc7b8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:17271, start at 2025-04-04 16:11:50.144057292 +0800 CST m=+3.690277980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:50.152 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:50.226 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:50.226 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367707515850467 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6cc7b8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:17271, start at 2025-04-04 16:11:50.144057292 +0800 CST m=+3.690277980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:50.152 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:50.226 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:50.226 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367700233582457 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6cc9e00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:17348, start at 2025-04-04 16:11:50.270525401 +0800 CST m=+3.643304053 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:50.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:50.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:50.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (12623) - No such process wait process 12623 exit for 1-th time... process 12623 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 18200 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367709560548565 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6cd0080002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:17197, start at 2025-04-04 16:11:50.659971406 +0800 CST m=+3.522460399 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:50.724 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:50.716 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:50.716 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367709560548565 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6cd0080002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:17197, start at 2025-04-04 16:11:50.659971406 +0800 CST m=+3.522460399 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:50.724 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:50.716 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:50.716 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367704995008554 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6cd280000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-90s3w--4dgm8, pid:17276, start at 2025-04-04 16:11:50.84407142 +0800 CST m=+3.589272176 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:50.857 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:50.916 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:50.916 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_J/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:51 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9c7c23d0-0a86-463a-b810-0a3b6b559904 {"id":"9c7c23d0-0a86-463a-b810-0a3b6b559904","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754308} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9c7c23d0-0a86-463a-b810-0a3b6b559904 {"id":"9c7c23d0-0a86-463a-b810-0a3b6b559904","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754308} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9c7c23d0-0a86-463a-b810-0a3b6b559904 {"id":"9c7c23d0-0a86-463a-b810-0a3b6b559904","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754308} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114729042673666 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.30393.out cli changefeed create --start-ts=457114729042673666 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367674212210317,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:51.736198826+08:00","start_ts":457114729042673666,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114729042673666,"checkpoint_ts":457114729042673666,"checkpoint_time":"2025-04-04 16:11:46.956","gid":{"low":7515052530135819116,"high":14478894183766776457}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x + ans='[ { "id": "040778c9-0001-42ce-bc0c-36cf3cac6f8e", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" }, { "id": "f0eb9874-49dc-4e56-9eae-d08f3bf53bf0", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"040778c9-0001-42ce-bc0c-36cf3cac6f8e",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '},' '{' '"id":' '"f0eb9874-49dc-4e56-9eae-d08f3bf53bf0",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=040778c9-0001-42ce-bc0c-36cf3cac6f8e + '[' -z 040778c9-0001-42ce-bc0c-36cf3cac6f8e ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 040778c9-0001-42ce-bc0c-36cf3cac6f8e + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.cli.19048.out cli changefeed move-table -c test -t 114 -d 040778c9-0001-42ce-bc0c-36cf3cac6f8e ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c82f854f-1294-43d2-b752-790c9c30ea54 {"id":"c82f854f-1294-43d2-b752-790c9c30ea54","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/capture/eb300a31-95d1-406d-acd2-a10831163b64 {"id":"eb300a31-95d1-406d-acd2-a10831163b64","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754309} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10830979984427073096,"high":7852405283103412372},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367627689970784,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:42.068524892+08:00","start-ts":457114726514819073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727760003077} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114728139849756,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c82f854f-1294-43d2-b752-790c9c30ea54 {"id":"c82f854f-1294-43d2-b752-790c9c30ea54","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/capture/eb300a31-95d1-406d-acd2-a10831163b64 {"id":"eb300a31-95d1-406d-acd2-a10831163b64","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754309} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10830979984427073096,"high":7852405283103412372},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367627689970784,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:42.068524892+08:00","start-ts":457114726514819073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727760003077} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114728139849756,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c82f854f-1294-43d2-b752-790c9c30ea54 {"id":"c82f854f-1294-43d2-b752-790c9c30ea54","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/capture/eb300a31-95d1-406d-acd2-a10831163b64 {"id":"eb300a31-95d1-406d-acd2-a10831163b64","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754309} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccafe9 d32e3b7a-2111-4b1e-a199-7f27b0b18e65 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb0cb c82f854f-1294-43d2-b752-790c9c30ea54 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10830979984427073096,"high":7852405283103412372},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367627689970784,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:42.068524892+08:00","start-ts":457114726514819073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727760003077} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114728139849756,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.16286.out cli capture list Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3668 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c857d8f-b9db-448f-a282-2480fc8ef793 {"id":"0c857d8f-b9db-448f-a282-2480fc8ef793","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/capture/188e9590-4393-4759-b85d-6e776363ce0d {"id":"188e9590-4393-4759-b85d-6e776363ce0d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754309} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac7614e 188e9590-4393-4759-b85d-6e776363ce0d /tidb/cdc/default/__cdc_meta__/owner/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac7614e 188e9590-4393-4759-b85d-6e776363ce0d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15512654085710704676,"high":14444012998446361230},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367628688494762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:40.705455139+08:00","start-ts":457114726158303233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727403487234} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727901560858,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c857d8f-b9db-448f-a282-2480fc8ef793 {"id":"0c857d8f-b9db-448f-a282-2480fc8ef793","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/capture/188e9590-4393-4759-b85d-6e776363ce0d {"id":"188e9590-4393-4759-b85d-6e776363ce0d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754309} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac7614e 188e9590-4393-4759-b85d-6e776363ce0d /tidb/cdc/default/__cdc_meta__/owner/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac7614e 188e9590-4393-4759-b85d-6e776363ce0d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15512654085710704676,"high":14444012998446361230},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367628688494762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:40.705455139+08:00","start-ts":457114726158303233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727403487234} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727901560858,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c857d8f-b9db-448f-a282-2480fc8ef793 {"id":"0c857d8f-b9db-448f-a282-2480fc8ef793","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/capture/188e9590-4393-4759-b85d-6e776363ce0d {"id":"188e9590-4393-4759-b85d-6e776363ce0d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754309} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac75fe3 87e48756-869f-494a-9e2a-9ce99a437fba /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac7614e 188e9590-4393-4759-b85d-6e776363ce0d /tidb/cdc/default/__cdc_meta__/owner/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac7614e 188e9590-4393-4759-b85d-6e776363ce0d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15512654085710704676,"high":14444012998446361230},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367628688494762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:40.705455139+08:00","start-ts":457114726158303233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727403487234} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114727901560858,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.16330.out cli capture list + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.15940.out cli tso query --pd=http://127.0.0.1:2379 + set +x Command succeeded on attempt 1 + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 table fail_over_ddl_test.test2 not exists for 1-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.cli.18690.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + ans='[ { "id": "c82f854f-1294-43d2-b752-790c9c30ea54", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "eb300a31-95d1-406d-acd2-a10831163b64", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"c82f854f-1294-43d2-b752-790c9c30ea54",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"eb300a31-95d1-406d-acd2-a10831163b64",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=eb300a31-95d1-406d-acd2-a10831163b64 + '[' -z eb300a31-95d1-406d-acd2-a10831163b64 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d eb300a31-95d1-406d-acd2-a10831163b64 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.16316.out cli changefeed move-table -c test -t 114 -d eb300a31-95d1-406d-acd2-a10831163b64 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.18502.out cli tso query --pd=http://127.0.0.1:2379 + set +x + ans='[ { "id": "0c857d8f-b9db-448f-a282-2480fc8ef793", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "188e9590-4393-4759-b85d-6e776363ce0d", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"0c857d8f-b9db-448f-a282-2480fc8ef793",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"188e9590-4393-4759-b85d-6e776363ce0d",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=188e9590-4393-4759-b85d-6e776363ce0d + '[' -z 188e9590-4393-4759-b85d-6e776363ce0d ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 188e9590-4393-4759-b85d-6e776363ce0d + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.16361.out cli changefeed move-table -c test -t 114 -d 188e9590-4393-4759-b85d-6e776363ce0d + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x + tso='457114730599546881 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114730599546881 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:54 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1597515977.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114730747658241 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114730747658241 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:54 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1873218734.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:54 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1294112943.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 exists + set +x + tso='457114730904682497 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114730904682497 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:11:55 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1854018542.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (30336) - No such process wait process 30336 exit for 1-th time... process 30336 already exit [Fri Apr 4 16:11:55 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.3046430466.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367731149877193 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6d20e00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:17142, start at 2025-04-04 16:11:55.83380932 +0800 CST m=+3.333204099 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:55.842 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:55.832 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:55.832 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367731149877193 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6d20e00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:17142, start at 2025-04-04 16:11:55.83380932 +0800 CST m=+3.333204099 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:55.842 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:55.832 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:55.832 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367728415544507 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6d2300000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-w6z00--g2hmb, pid:17217, start at 2025-04-04 16:11:55.994065404 +0800 CST m=+3.337943718 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:56.040 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:56.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:56.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_F/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_B Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:57 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2165c272-0dc3-4d68-8453-0b0d8b645467 {"id":"2165c272-0dc3-4d68-8453-0b0d8b645467","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754314} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2165c272-0dc3-4d68-8453-0b0d8b645467 {"id":"2165c272-0dc3-4d68-8453-0b0d8b645467","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754314} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2165c272-0dc3-4d68-8453-0b0d8b645467 {"id":"2165c272-0dc3-4d68-8453-0b0d8b645467","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754314} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114730599546881 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.16029.out cli changefeed create --start-ts=457114730599546881 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367703208224136,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:57.700031741+08:00","start_ts":457114730599546881,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114730599546881,"checkpoint_ts":457114730599546881,"checkpoint_time":"2025-04-04 16:11:52.895","gid":{"low":8811465680906879878,"high":13686026516719565716}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:57 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 {"id":"a4732785-1a1c-4cc5-8e1f-3655a2fdfc14","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754315} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 {"id":"a4732785-1a1c-4cc5-8e1f-3655a2fdfc14","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754315} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 {"id":"a4732785-1a1c-4cc5-8e1f-3655a2fdfc14","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754315} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114730747658241 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.cli.18803.out cli changefeed create --start-ts=457114730747658241 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367707515850467,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:58.275957437+08:00","start_ts":457114730747658241,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114730747658241,"checkpoint_ts":457114730747658241,"checkpoint_time":"2025-04-04 16:11:53.460","gid":{"low":2830846838561264130,"high":10486332291294166706}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52132311-d7f7-45cb-80f9-26ccb7dde5d8 {"id":"52132311-d7f7-45cb-80f9-26ccb7dde5d8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754315} /tidb/cdc/default/__cdc_meta__/capture/a97d8c28-1fef-45f3-9e80-4fee4162f40f {"id":"a97d8c28-1fef-45f3-9e80-4fee4162f40f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754286} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71af24 52132311-d7f7-45cb-80f9-26ccb7dde5d8 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/__cdc_meta__/owner/223195ffda71af24 52132311-d7f7-45cb-80f9-26ccb7dde5d8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9605402568531767544,"high":8153563322789178034},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367536119683961,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:18.745022759+08:00","start-ts":457114720393494529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721638678537} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725426921475,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52132311-d7f7-45cb-80f9-26ccb7dde5d8 {"id":"52132311-d7f7-45cb-80f9-26ccb7dde5d8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754315} /tidb/cdc/default/__cdc_meta__/capture/a97d8c28-1fef-45f3-9e80-4fee4162f40f {"id":"a97d8c28-1fef-45f3-9e80-4fee4162f40f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754286} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71af24 52132311-d7f7-45cb-80f9-26ccb7dde5d8 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/__cdc_meta__/owner/223195ffda71af24 52132311-d7f7-45cb-80f9-26ccb7dde5d8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9605402568531767544,"high":8153563322789178034},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367536119683961,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:18.745022759+08:00","start-ts":457114720393494529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721638678537} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725426921475,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52132311-d7f7-45cb-80f9-26ccb7dde5d8 {"id":"52132311-d7f7-45cb-80f9-26ccb7dde5d8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754315} /tidb/cdc/default/__cdc_meta__/capture/a97d8c28-1fef-45f3-9e80-4fee4162f40f {"id":"a97d8c28-1fef-45f3-9e80-4fee4162f40f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754286} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71a9c2 6ff5c030-4c8e-45f6-86ba-98afa63d7566 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda71af24 52132311-d7f7-45cb-80f9-26ccb7dde5d8 /tidb/cdc/default/__cdc_meta__/owner/223195ffda71aa28 a97d8c28-1fef-45f3-9e80-4fee4162f40f /tidb/cdc/default/__cdc_meta__/owner/223195ffda71af24 52132311-d7f7-45cb-80f9-26ccb7dde5d8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9605402568531767544,"high":8153563322789178034},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367536119683961,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:18.745022759+08:00","start-ts":457114720393494529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114721638678537} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114725426921475,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:58 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c35bac40-e946-451d-9b0f-43a26fdd52ce {"id":"c35bac40-e946-451d-9b0f-43a26fdd52ce","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754315} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c35bac40-e946-451d-9b0f-43a26fdd52ce {"id":"c35bac40-e946-451d-9b0f-43a26fdd52ce","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754315} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c35bac40-e946-451d-9b0f-43a26fdd52ce {"id":"c35bac40-e946-451d-9b0f-43a26fdd52ce","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754315} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114730904682497 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.18601.out cli changefeed create --start-ts=457114730904682497 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367709560548565,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:11:58.789451075+08:00","start_ts":457114730904682497,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114730904682497,"checkpoint_ts":457114730904682497,"checkpoint_time":"2025-04-04 16:11:54.059","gid":{"low":4128942869739245599,"high":4146714913998868127}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:11:58 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 {"id":"bdfd02be-7cf5-4e43-a6cc-23df5ba231c4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754316} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114730667966494,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 {"id":"bdfd02be-7cf5-4e43-a6cc-23df5ba231c4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754316} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114730667966494,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 {"id":"bdfd02be-7cf5-4e43-a6cc-23df5ba231c4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754316} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114730667966494,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:11:58 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.3054130543.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Command succeeded on attempt 1 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully table fail_over_ddl_test.test2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.18507.out cli tso query --pd=http://127.0.0.1:2379 + set +x Command succeeded on attempt 1 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully table fail_over_ddl_test.test2 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x Command succeeded on attempt 1 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later table fail_over_ddl_test.test2 not exists for 1-th check, retry later + set +x + tso='457114732336775169 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114732336775169 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:12:00 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1853618538.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15980) - No such process wait process 15980 exit for 1-th time... process 15980 already exit [Fri Apr 4 16:12:01 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1610416106.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table fail_over_ddl_test.test2 exists table fail_over_ddl_test.test2 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bce15f27-8ce4-485b-afdd-6c0a969042b6 {"id":"bce15f27-8ce4-485b-afdd-6c0a969042b6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754319} /tidb/cdc/default/__cdc_meta__/capture/bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 {"id":"bdfd02be-7cf5-4e43-a6cc-23df5ba231c4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754316} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44e28 bce15f27-8ce4-485b-afdd-6c0a969042b6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44e28 bce15f27-8ce4-485b-afdd-6c0a969042b6 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114730667966494,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bce15f27-8ce4-485b-afdd-6c0a969042b6 {"id":"bce15f27-8ce4-485b-afdd-6c0a969042b6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754319} /tidb/cdc/default/__cdc_meta__/capture/bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 {"id":"bdfd02be-7cf5-4e43-a6cc-23df5ba231c4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754316} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44e28 bce15f27-8ce4-485b-afdd-6c0a969042b6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44e28 bce15f27-8ce4-485b-afdd-6c0a969042b6 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114730667966494,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bce15f27-8ce4-485b-afdd-6c0a969042b6 {"id":"bce15f27-8ce4-485b-afdd-6c0a969042b6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754319} /tidb/cdc/default/__cdc_meta__/capture/bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 {"id":"bdfd02be-7cf5-4e43-a6cc-23df5ba231c4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754316} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44ce6 9c7c23d0-0a86-463a-b810-0a3b6b559904 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44e28 bce15f27-8ce4-485b-afdd-6c0a969042b6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf44e28 bce15f27-8ce4-485b-afdd-6c0a969042b6 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114730667966494,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.30603.out cli capture list /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18737) - No such process wait process 18737 exit for 1-th time... process 18737 already exit [Fri Apr 4 16:12:02 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1887318875.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18545) - No such process wait process 18545 exit for 1-th time... process 18545 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:12:02 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1868718689.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + ans='[ { "id": "bdfd02be-7cf5-4e43-a6cc-23df5ba231c4", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "bce15f27-8ce4-485b-afdd-6c0a969042b6", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"bdfd02be-7cf5-4e43-a6cc-23df5ba231c4",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"bce15f27-8ce4-485b-afdd-6c0a969042b6",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=bce15f27-8ce4-485b-afdd-6c0a969042b6 + '[' -z bce15f27-8ce4-485b-afdd-6c0a969042b6 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d bce15f27-8ce4-485b-afdd-6c0a969042b6 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.30637.out cli changefeed move-table -c test -t 114 -d bce15f27-8ce4-485b-afdd-6c0a969042b6 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367668127296815 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6d94f4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:15197, start at 2025-04-04 16:12:03.276056783 +0800 CST m=+24.102074649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:03.284 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:03.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:03.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367668127296815 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6d94f4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:15197, start at 2025-04-04 16:12:03.276056783 +0800 CST m=+24.102074649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:03.284 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:03.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:03.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367673851154831 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6c4118000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-bjkgt--5xn5b, pid:15276, start at 2025-04-04 16:11:41.552674343 +0800 CST m=+2.209043190 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:13:41.560 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:11:41.560 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:01:41.560 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_K/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:03 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/014891ce-25ad-44e7-a455-6ec5f0183c99 {"id":"014891ce-25ad-44e7-a455-6ec5f0183c99","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754321} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/014891ce-25ad-44e7-a455-6ec5f0183c99 {"id":"014891ce-25ad-44e7-a455-6ec5f0183c99","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754321} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/014891ce-25ad-44e7-a455-6ec5f0183c99 {"id":"014891ce-25ad-44e7-a455-6ec5f0183c99","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754321} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114732336775169 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.18605.out cli changefeed create --start-ts=457114732336775169 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367731149877193,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:12:04.327121447+08:00","start_ts":457114732336775169,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114732336775169,"checkpoint_ts":457114732336775169,"checkpoint_time":"2025-04-04 16:11:59.522","gid":{"low":15367819270581404014,"high":4849217813893219260}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16510) - No such process wait process 16510 exit for 1-th time... process 16510 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16582) - No such process wait process 16582 exit for 1-th time... process 16582 already exit [Fri Apr 4 16:12:03 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1671116713.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 {"id":"e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8811465680906879878,"high":13686026516719565716},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367703208224136,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:57.700031741+08:00","start-ts":457114730599546881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114731857838084} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732211732499,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 {"id":"e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8811465680906879878,"high":13686026516719565716},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367703208224136,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:57.700031741+08:00","start-ts":457114730599546881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114731857838084} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732211732499,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 {"id":"e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8811465680906879878,"high":13686026516719565716},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367703208224136,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:57.700031741+08:00","start-ts":457114730599546881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114731857838084} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732211732499,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:04 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1618416186.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa9286d1-b4e1-4336-90bc-5f4873f52161 {"id":"fa9286d1-b4e1-4336-90bc-5f4873f52161","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732359843845,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa9286d1-b4e1-4336-90bc-5f4873f52161 {"id":"fa9286d1-b4e1-4336-90bc-5f4873f52161","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732359843845,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa9286d1-b4e1-4336-90bc-5f4873f52161 {"id":"fa9286d1-b4e1-4336-90bc-5f4873f52161","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114732359843845,"progress":1}' + break + set +x [Fri Apr 4 16:12:05 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1894918951.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2182464-08bf-417a-b0b0-d7ec30a70767 {"id":"e2182464-08bf-417a-b0b0-d7ec30a70767","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754323} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4128942869739245599,"high":4146714913998868127},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367709560548565,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.789451075+08:00","start-ts":457114730904682497,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732136759304} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732529975297,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2182464-08bf-417a-b0b0-d7ec30a70767 {"id":"e2182464-08bf-417a-b0b0-d7ec30a70767","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754323} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4128942869739245599,"high":4146714913998868127},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367709560548565,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.789451075+08:00","start-ts":457114730904682497,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732136759304} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732529975297,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2182464-08bf-417a-b0b0-d7ec30a70767 {"id":"e2182464-08bf-417a-b0b0-d7ec30a70767","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754323} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4128942869739245599,"high":4146714913998868127},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367709560548565,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.789451075+08:00","start-ts":457114730904682497,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732136759304} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732529975297,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1375 0 --:--:-- --:--:-- --:--:-- 1375 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:05 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1875618758.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.cli.16516.out cli tso query --pd=http://127.0.0.1:2379 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367775215902593 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6dbb68000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:16580, start at 2025-04-04 16:12:05.758223473 +0800 CST m=+3.496797209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:05.819 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:05.822 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:05.822 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367775215902593 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6dbb68000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:16580, start at 2025-04-04 16:12:05.758223473 +0800 CST m=+3.496797209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:05.819 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:05.822 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:05.822 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367775532686554 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6dbe880005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:16636, start at 2025-04-04 16:12:05.930095646 +0800 CST m=+3.570793295 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:05.940 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:05.922 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:05.922 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47626f3a-e3cd-42bd-9bac-2586f97b21fd {"id":"47626f3a-e3cd-42bd-9bac-2586f97b21fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754324} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732671795207,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47626f3a-e3cd-42bd-9bac-2586f97b21fd {"id":"47626f3a-e3cd-42bd-9bac-2586f97b21fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754324} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732671795207,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47626f3a-e3cd-42bd-9bac-2586f97b21fd {"id":"47626f3a-e3cd-42bd-9bac-2586f97b21fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754324} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8e6c6 5e4e06d2-e41a-4030-b2bc-a0dd86e92239 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732671795207,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:06 CST 2025] <<<<<< START cdc server in fail_over_ddl_D case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_D.1678616788.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_D/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_D/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114734037827586 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114734037827586 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:12:07 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1655416556.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:07 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 {"id":"e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/capture/e6dc4224-3cfb-48d5-a55d-20e62df677c3 {"id":"e6dc4224-3cfb-48d5-a55d-20e62df677c3","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754325} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb429 e6dc4224-3cfb-48d5-a55d-20e62df677c3 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb429 e6dc4224-3cfb-48d5-a55d-20e62df677c3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8811465680906879878,"high":13686026516719565716},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367703208224136,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:57.700031741+08:00","start-ts":457114730599546881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114731857838084} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732211732499,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 {"id":"e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/capture/e6dc4224-3cfb-48d5-a55d-20e62df677c3 {"id":"e6dc4224-3cfb-48d5-a55d-20e62df677c3","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754325} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb429 e6dc4224-3cfb-48d5-a55d-20e62df677c3 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb429 e6dc4224-3cfb-48d5-a55d-20e62df677c3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8811465680906879878,"high":13686026516719565716},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367703208224136,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:57.700031741+08:00","start-ts":457114730599546881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114731857838084} /tidb/cdc/default/default/changefeed/status/test + grep -q 'failed to get info:' {"checkpoint-ts":457114732211732499,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 {"id":"e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/capture/e6dc4224-3cfb-48d5-a55d-20e62df677c3 {"id":"e6dc4224-3cfb-48d5-a55d-20e62df677c3","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754325} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb2e6 2165c272-0dc3-4d68-8453-0b0d8b645467 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb429 e6dc4224-3cfb-48d5-a55d-20e62df677c3 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb429 e6dc4224-3cfb-48d5-a55d-20e62df677c3 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8811465680906879878,"high":13686026516719565716},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367703208224136,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:57.700031741+08:00","start-ts":457114730599546881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114731857838084} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732211732499,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.16257.out cli capture list table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18541) - No such process wait process 18541 exit for 1-th time... process 18541 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:12:08 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1867718681.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.17933.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ce4ae18-54d6-4464-8f9e-2552d8a654fd {"id":"6ce4ae18-54d6-4464-8f9e-2552d8a654fd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754325} /tidb/cdc/default/__cdc_meta__/capture/fa9286d1-b4e1-4336-90bc-5f4873f52161 {"id":"fa9286d1-b4e1-4336-90bc-5f4873f52161","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116c27 6ce4ae18-54d6-4464-8f9e-2552d8a654fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116c27 6ce4ae18-54d6-4464-8f9e-2552d8a654fd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732399165454,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ce4ae18-54d6-4464-8f9e-2552d8a654fd {"id":"6ce4ae18-54d6-4464-8f9e-2552d8a654fd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754325} /tidb/cdc/default/__cdc_meta__/capture/fa9286d1-b4e1-4336-90bc-5f4873f52161 {"id":"fa9286d1-b4e1-4336-90bc-5f4873f52161","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116c27 6ce4ae18-54d6-4464-8f9e-2552d8a654fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116c27 6ce4ae18-54d6-4464-8f9e-2552d8a654fd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732399165454,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ce4ae18-54d6-4464-8f9e-2552d8a654fd {"id":"6ce4ae18-54d6-4464-8f9e-2552d8a654fd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754325} /tidb/cdc/default/__cdc_meta__/capture/fa9286d1-b4e1-4336-90bc-5f4873f52161 {"id":"fa9286d1-b4e1-4336-90bc-5f4873f52161","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116ae3 a4732785-1a1c-4cc5-8e1f-3655a2fdfc14 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116c27 6ce4ae18-54d6-4464-8f9e-2552d8a654fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116c27 6ce4ae18-54d6-4464-8f9e-2552d8a654fd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732399165454,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 {"id":"4a48b3cc-5ca4-47f7-b51d-55110fd2ea91","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754326} /tidb/cdc/default/__cdc_meta__/capture/e2182464-08bf-417a-b0b0-d7ec30a70767 {"id":"e2182464-08bf-417a-b0b0-d7ec30a70767","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754323} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb123428 4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb123428 4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4128942869739245599,"high":4146714913998868127},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367709560548565,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.789451075+08:00","start-ts":457114730904682497,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732136759304} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732529975297,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 {"id":"4a48b3cc-5ca4-47f7-b51d-55110fd2ea91","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754326} /tidb/cdc/default/__cdc_meta__/capture/e2182464-08bf-417a-b0b0-d7ec30a70767 {"id":"e2182464-08bf-417a-b0b0-d7ec30a70767","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754323} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb123428 4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb123428 4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4128942869739245599,"high":4146714913998868127},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367709560548565,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.789451075+08:00","start-ts":457114730904682497,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732136759304} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732529975297,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 {"id":"4a48b3cc-5ca4-47f7-b51d-55110fd2ea91","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754326} /tidb/cdc/default/__cdc_meta__/capture/e2182464-08bf-417a-b0b0-d7ec30a70767 {"id":"e2182464-08bf-417a-b0b0-d7ec30a70767","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754323} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1232e6 c35bac40-e946-451d-9b0f-43a26fdd52ce /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb123428 4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb123428 4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4128942869739245599,"high":4146714913998868127},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367709560548565,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.789451075+08:00","start-ts":457114730904682497,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732136759304} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114732529975297,"progress":1}' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.18823.out cli capture list + set +x + ans='[ { "id": "e6dc4224-3cfb-48d5-a55d-20e62df677c3", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" }, { "id": "e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"e6dc4224-3cfb-48d5-a55d-20e62df677c3",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '},' '{' '"id":' '"e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=e6dc4224-3cfb-48d5-a55d-20e62df677c3 + '[' -z e6dc4224-3cfb-48d5-a55d-20e62df677c3 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d e6dc4224-3cfb-48d5-a55d-20e62df677c3 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.16288.out cli changefeed move-table -c test -t 114 -d e6dc4224-3cfb-48d5-a55d-20e62df677c3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:09 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f8617f0-7020-472b-9f26-9a2df5ec02fd {"id":"1f8617f0-7020-472b-9f26-9a2df5ec02fd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754327} /tidb/cdc/default/__cdc_meta__/capture/47626f3a-e3cd-42bd-9bac-2586f97b21fd {"id":"47626f3a-e3cd-42bd-9bac-2586f97b21fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754324} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8ec63 1f8617f0-7020-472b-9f26-9a2df5ec02fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8ec63 1f8617f0-7020-472b-9f26-9a2df5ec02fd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732671795207,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f8617f0-7020-472b-9f26-9a2df5ec02fd {"id":"1f8617f0-7020-472b-9f26-9a2df5ec02fd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754327} /tidb/cdc/default/__cdc_meta__/capture/47626f3a-e3cd-42bd-9bac-2586f97b21fd {"id":"47626f3a-e3cd-42bd-9bac-2586f97b21fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754324} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8ec63 1f8617f0-7020-472b-9f26-9a2df5ec02fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8ec63 1f8617f0-7020-472b-9f26-9a2df5ec02fd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732671795207,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f8617f0-7020-472b-9f26-9a2df5ec02fd {"id":"1f8617f0-7020-472b-9f26-9a2df5ec02fd","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754327} /tidb/cdc/default/__cdc_meta__/capture/47626f3a-e3cd-42bd-9bac-2586f97b21fd {"id":"47626f3a-e3cd-42bd-9bac-2586f97b21fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754324} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaa8ec63 1f8617f0-7020-472b-9f26-9a2df5ec02fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8ec01 47626f3a-e3cd-42bd-9bac-2586f97b21fd /tidb/cdc/default/__cdc_meta__/owner/223195ffdaa8ec63 1f8617f0-7020-472b-9f26-9a2df5ec02fd /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16233957215137547470,"high":603714472361012659},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367584068670880,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:30.433968958+08:00","start-ts":457114723457433601,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114724702879752} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114732671795207,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully check diff failed 1-th time, retry later + set +x + tso='457114734879834113 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114734879834113 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:12:10 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1796817970.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + ans='[ { "id": "e2182464-08bf-417a-b0b0-d7ec30a70767", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "4a48b3cc-5ca4-47f7-b51d-55110fd2ea91", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"e2182464-08bf-417a-b0b0-d7ec30a70767",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"4a48b3cc-5ca4-47f7-b51d-55110fd2ea91",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 + '[' -z 4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.cli.18853.out cli changefeed move-table -c test -t 114 -d 4a48b3cc-5ca4-47f7-b51d-55110fd2ea91 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:10 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c51d653-7e95-412e-8ce6-5e25a6b25c3f {"id":"6c51d653-7e95-412e-8ce6-5e25a6b25c3f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754327} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c51d653-7e95-412e-8ce6-5e25a6b25c3f {"id":"6c51d653-7e95-412e-8ce6-5e25a6b25c3f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754327} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c51d653-7e95-412e-8ce6-5e25a6b25c3f {"id":"6c51d653-7e95-412e-8ce6-5e25a6b25c3f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754327} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114734037827586 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.cli.16613.out cli changefeed create --start-ts=457114734037827586 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367668127296815,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:12:10.790597342+08:00","start_ts":457114734037827586,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114734037827586,"checkpoint_ts":457114734037827586,"checkpoint_time":"2025-04-04 16:12:06.011","gid":{"low":1751658282563072813,"high":7566144674979320471}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' run task successfully check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18878) - No such process wait process 18878 exit for 1-th time... process 18878 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16149) - No such process wait process 16149 exit for 1-th time... process 16149 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18954) - No such process wait process 18954 exit for 1-th time... process 18954 already exit [Fri Apr 4 16:12:11 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1905719059.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49ab4092-c8ae-40bb-9609-d40b0f41aa7b {"id":"49ab4092-c8ae-40bb-9609-d40b0f41aa7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754328} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15367819270581404014,"high":4849217813893219260},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367731149877193,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:04.327121447+08:00","start-ts":457114732336775169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114733594804229} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114733974913052,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49ab4092-c8ae-40bb-9609-d40b0f41aa7b {"id":"49ab4092-c8ae-40bb-9609-d40b0f41aa7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754328} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15367819270581404014,"high":4849217813893219260},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367731149877193,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:04.327121447+08:00","start-ts":457114732336775169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114733594804229} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114733974913052,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49ab4092-c8ae-40bb-9609-d40b0f41aa7b {"id":"49ab4092-c8ae-40bb-9609-d40b0f41aa7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754328} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15367819270581404014,"high":4849217813893219260},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367731149877193,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:04.327121447+08:00","start-ts":457114732336775169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114733594804229} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114733974913052,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:11 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1875218754.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Command succeeded on attempt 1 check diff failed 2-th time, retry later table fail_over_ddl_test.test2 not exists for 1-th check, retry later + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 check diff failed 2-th time, retry later + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:13 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/133e6d36-747b-4a91-8b35-4cf69f53c198 {"id":"133e6d36-747b-4a91-8b35-4cf69f53c198","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754330} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/133e6d36-747b-4a91-8b35-4cf69f53c198 {"id":"133e6d36-747b-4a91-8b35-4cf69f53c198","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754330} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/133e6d36-747b-4a91-8b35-4cf69f53c198 {"id":"133e6d36-747b-4a91-8b35-4cf69f53c198","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754330} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114734879834113 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.18035.out cli changefeed create --start-ts=457114734879834113 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367775215902593,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:12:13.986016533+08:00","start_ts":457114734879834113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114734879834113,"checkpoint_ts":457114734879834113,"checkpoint_time":"2025-04-04 16:12:09.223","gid":{"low":3120498355177252900,"high":13612266207749673379}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table fail_over_ddl_test.test2 exists check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=4 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":4}' + echo '{"count":4}' {"count":4} ++ echo '{"count":4}' ++ jq -r .count + value=4 + '[' 4 == -1 ']' + '[' 4 == 4 ']' + exit 0 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16559) - No such process wait process 16559 exit for 1-th time... process 16559 already exit [Fri Apr 4 16:12:14 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1668916691.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24d30ad8-0c61-4430-a9f9-965984818306 {"id":"24d30ad8-0c61-4430-a9f9-965984818306","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754332} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734456995853,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24d30ad8-0c61-4430-a9f9-965984818306 {"id":"24d30ad8-0c61-4430-a9f9-965984818306","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754332} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734456995853,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24d30ad8-0c61-4430-a9f9-965984818306 {"id":"24d30ad8-0c61-4430-a9f9-965984818306","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754332} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734456995853,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:14 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.1914019142.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49ab4092-c8ae-40bb-9609-d40b0f41aa7b {"id":"49ab4092-c8ae-40bb-9609-d40b0f41aa7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754328} /tidb/cdc/default/__cdc_meta__/capture/6b2a6952-1948-42c6-9369-bb34004052a2 {"id":"6b2a6952-1948-42c6-9369-bb34004052a2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754331} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb291926 6b2a6952-1948-42c6-9369-bb34004052a2 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/owner/223195ffdb291926 6b2a6952-1948-42c6-9369-bb34004052a2 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15367819270581404014,"high":4849217813893219260},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367731149877193,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:04.327121447+08:00","start-ts":457114732336775169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114733594804229} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114733974913052,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49ab4092-c8ae-40bb-9609-d40b0f41aa7b {"id":"49ab4092-c8ae-40bb-9609-d40b0f41aa7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754328} /tidb/cdc/default/__cdc_meta__/capture/6b2a6952-1948-42c6-9369-bb34004052a2 {"id":"6b2a6952-1948-42c6-9369-bb34004052a2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754331} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb291926 6b2a6952-1948-42c6-9369-bb34004052a2 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/owner/223195ffdb291926 6b2a6952-1948-42c6-9369-bb34004052a2 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15367819270581404014,"high":4849217813893219260},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367731149877193,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:04.327121447+08:00","start-ts":457114732336775169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114733594804229} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114733974913052,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49ab4092-c8ae-40bb-9609-d40b0f41aa7b {"id":"49ab4092-c8ae-40bb-9609-d40b0f41aa7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754328} /tidb/cdc/default/__cdc_meta__/capture/6b2a6952-1948-42c6-9369-bb34004052a2 {"id":"6b2a6952-1948-42c6-9369-bb34004052a2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754331} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2917e2 014891ce-25ad-44e7-a455-6ec5f0183c99 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb291926 6b2a6952-1948-42c6-9369-bb34004052a2 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/owner/223195ffdb291926 6b2a6952-1948-42c6-9369-bb34004052a2 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15367819270581404014,"high":4849217813893219260},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367731149877193,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:04.327121447+08:00","start-ts":457114732336775169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114733594804229} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114733974913052,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.18829.out cli capture list cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseD-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_D/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_D/stdout0-2.log /tmp/tidb_cdc_test/fail_over_ddl_D/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_D/stdout1-1.log /tmp/tidb_cdc_test/fail_over_ddl_D/stdout1-2.log no DATA RACE found [Fri Apr 4 16:12:14 CST 2025] <<<<<< run test case fail_over_ddl_D success! >>>>>> cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseN-2 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x Command succeeded on attempt 1 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:15 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1642616428.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + ans='[ { "id": "49ab4092-c8ae-40bb-9609-d40b0f41aa7b", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "6b2a6952-1948-42c6-9369-bb34004052a2", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"49ab4092-c8ae-40bb-9609-d40b0f41aa7b",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"6b2a6952-1948-42c6-9369-bb34004052a2",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=6b2a6952-1948-42c6-9369-bb34004052a2 + '[' -z 6b2a6952-1948-42c6-9369-bb34004052a2 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 6b2a6952-1948-42c6-9369-bb34004052a2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.cli.18862.out cli changefeed move-table -c test -t 114 -d 6b2a6952-1948-42c6-9369-bb34004052a2 + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:14 CST 2025] <<<<<< START cdc server in fail_over_ddl_H case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_H.1446114463.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_H/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_H/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 not exists for 1-th check, retry later command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16267) - No such process wait process 16267 exit for 1-th time... process 16267 already exit [Fri Apr 4 16:12:14 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1645516457.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3565 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c857d8f-b9db-448f-a282-2480fc8ef793 {"id":"0c857d8f-b9db-448f-a282-2480fc8ef793","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/capture/fac90366-8e31-47aa-816a-831691540def {"id":"fac90366-8e31-47aa-816a-831691540def","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac7662b fac90366-8e31-47aa-816a-831691540def /tidb/cdc/default/__cdc_meta__/owner/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac7662b fac90366-8e31-47aa-816a-831691540def /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15512654085710704676,"high":14444012998446361230},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367628688494762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:40.705455139+08:00","start-ts":457114726158303233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727403487234} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735635070982,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c857d8f-b9db-448f-a282-2480fc8ef793 {"id":"0c857d8f-b9db-448f-a282-2480fc8ef793","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/capture/fac90366-8e31-47aa-816a-831691540def {"id":"fac90366-8e31-47aa-816a-831691540def","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac7662b fac90366-8e31-47aa-816a-831691540def /tidb/cdc/default/__cdc_meta__/owner/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac7662b fac90366-8e31-47aa-816a-831691540def /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15512654085710704676,"high":14444012998446361230},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367628688494762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:40.705455139+08:00","start-ts":457114726158303233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727403487234} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735635070982,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c857d8f-b9db-448f-a282-2480fc8ef793 {"id":"0c857d8f-b9db-448f-a282-2480fc8ef793","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754306} /tidb/cdc/default/__cdc_meta__/capture/fac90366-8e31-47aa-816a-831691540def {"id":"fac90366-8e31-47aa-816a-831691540def","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac7662b fac90366-8e31-47aa-816a-831691540def /tidb/cdc/default/__cdc_meta__/owner/223195ffdac760ec 0c857d8f-b9db-448f-a282-2480fc8ef793 /tidb/cdc/default/__cdc_meta__/owner/223195ffdac7662b fac90366-8e31-47aa-816a-831691540def /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15512654085710704676,"high":14444012998446361230},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367628688494762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:40.705455139+08:00","start-ts":457114726158303233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727403487234} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735635070982,"progress":1}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f357a362-09a5-4a19-a660-ca681afd5ce8 {"id":"f357a362-09a5-4a19-a660-ca681afd5ce8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735676227608,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f357a362-09a5-4a19-a660-ca681afd5ce8 {"id":"f357a362-09a5-4a19-a660-ca681afd5ce8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735676227608,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f357a362-09a5-4a19-a660-ca681afd5ce8 {"id":"f357a362-09a5-4a19-a660-ca681afd5ce8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735676227608,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:12:17 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1676016762.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24d30ad8-0c61-4430-a9f9-965984818306 {"id":"24d30ad8-0c61-4430-a9f9-965984818306","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754332} /tidb/cdc/default/__cdc_meta__/capture/9b289f07-8875-4529-bd26-6eee45bc0a73 {"id":"9b289f07-8875-4529-bd26-6eee45bc0a73","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116d9f 9b289f07-8875-4529-bd26-6eee45bc0a73 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116d9f 9b289f07-8875-4529-bd26-6eee45bc0a73 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734456995853,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24d30ad8-0c61-4430-a9f9-965984818306 {"id":"24d30ad8-0c61-4430-a9f9-965984818306","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754332} /tidb/cdc/default/__cdc_meta__/capture/9b289f07-8875-4529-bd26-6eee45bc0a73 {"id":"9b289f07-8875-4529-bd26-6eee45bc0a73","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116d9f 9b289f07-8875-4529-bd26-6eee45bc0a73 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116d9f 9b289f07-8875-4529-bd26-6eee45bc0a73 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734456995853,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24d30ad8-0c61-4430-a9f9-965984818306 {"id":"24d30ad8-0c61-4430-a9f9-965984818306","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754332} /tidb/cdc/default/__cdc_meta__/capture/9b289f07-8875-4529-bd26-6eee45bc0a73 {"id":"9b289f07-8875-4529-bd26-6eee45bc0a73","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116bc4 fa9286d1-b4e1-4336-90bc-5f4873f52161 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb116d9f 9b289f07-8875-4529-bd26-6eee45bc0a73 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116d40 24d30ad8-0c61-4430-a9f9-965984818306 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb116d9f 9b289f07-8875-4529-bd26-6eee45bc0a73 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2830846838561264130,"high":10486332291294166706},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367707515850467,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.275957437+08:00","start-ts":457114730747658241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732005687301} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734456995853,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/161c58bd-4518-4bd6-8d42-d6561688b92a {"id":"161c58bd-4518-4bd6-8d42-d6561688b92a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/capture/a52efcc0-736d-4773-b817-f79548b56849 {"id":"a52efcc0-736d-4773-b817-f79548b56849","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754293} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c58ac 161c58bd-4518-4bd6-8d42-d6561688b92a /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c58ac 161c58bd-4518-4bd6-8d42-d6561688b92a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12341241844399578707,"high":7306502954943539847},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367559394130666,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:25.597975129+08:00","start-ts":457114722195210241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114723440656389} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114736141271043,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/161c58bd-4518-4bd6-8d42-d6561688b92a {"id":"161c58bd-4518-4bd6-8d42-d6561688b92a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/capture/a52efcc0-736d-4773-b817-f79548b56849 {"id":"a52efcc0-736d-4773-b817-f79548b56849","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754293} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c58ac 161c58bd-4518-4bd6-8d42-d6561688b92a /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c58ac 161c58bd-4518-4bd6-8d42-d6561688b92a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12341241844399578707,"high":7306502954943539847},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367559394130666,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:25.597975129+08:00","start-ts":457114722195210241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114723440656389} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114736141271043,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/161c58bd-4518-4bd6-8d42-d6561688b92a {"id":"161c58bd-4518-4bd6-8d42-d6561688b92a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/capture/a52efcc0-736d-4773-b817-f79548b56849 {"id":"a52efcc0-736d-4773-b817-f79548b56849","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754293} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffda8c58ac 161c58bd-4518-4bd6-8d42-d6561688b92a /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c5327 a52efcc0-736d-4773-b817-f79548b56849 /tidb/cdc/default/__cdc_meta__/owner/223195ffda8c58ac 161c58bd-4518-4bd6-8d42-d6561688b92a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":12341241844399578707,"high":7306502954943539847},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367559394130666,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:25.597975129+08:00","start-ts":457114722195210241,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114723440656389} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114736141271043,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18874) - No such process wait process 18874 exit for 1-th time... process 18874 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 18200 0 --:--:-- --:--:-- --:--:-- 18200 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4fb68d5-7174-4553-a02c-5a0b287cb214 {"id":"e4fb68d5-7174-4553-a02c-5a0b287cb214","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/capture/eb300a31-95d1-406d-acd2-a10831163b64 {"id":"eb300a31-95d1-406d-acd2-a10831163b64","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754309} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb59d e4fb68d5-7174-4553-a02c-5a0b287cb214 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb59d e4fb68d5-7174-4553-a02c-5a0b287cb214 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10830979984427073096,"high":7852405283103412372},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367627689970784,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:42.068524892+08:00","start-ts":457114726514819073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727760003077} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114731547983880,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4fb68d5-7174-4553-a02c-5a0b287cb214 {"id":"e4fb68d5-7174-4553-a02c-5a0b287cb214","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/capture/eb300a31-95d1-406d-acd2-a10831163b64 {"id":"eb300a31-95d1-406d-acd2-a10831163b64","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754309} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb59d e4fb68d5-7174-4553-a02c-5a0b287cb214 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb59d e4fb68d5-7174-4553-a02c-5a0b287cb214 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10830979984427073096,"high":7852405283103412372},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367627689970784,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:42.068524892+08:00","start-ts":457114726514819073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727760003077} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114731547983880,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4fb68d5-7174-4553-a02c-5a0b287cb214 {"id":"e4fb68d5-7174-4553-a02c-5a0b287cb214","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/capture/eb300a31-95d1-406d-acd2-a10831163b64 {"id":"eb300a31-95d1-406d-acd2-a10831163b64","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754309} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaccb59d e4fb68d5-7174-4553-a02c-5a0b287cb214 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb12c eb300a31-95d1-406d-acd2-a10831163b64 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaccb59d e4fb68d5-7174-4553-a02c-5a0b287cb214 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":10830979984427073096,"high":7852405283103412372},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367627689970784,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:42.068524892+08:00","start-ts":457114726514819073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727760003077} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114731547983880,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table fail_over_ddl_test.test2 exists chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (17973) - No such process wait process 17973 exit for 1-th time... process 17973 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:12:19 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1811218114.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 2-th time, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d17ff97-bf7e-4ca4-a9a7-803f94b44843 {"id":"8d17ff97-bf7e-4ca4-a9a7-803f94b44843","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754338} /tidb/cdc/default/__cdc_meta__/capture/f357a362-09a5-4a19-a660-ca681afd5ce8 {"id":"f357a362-09a5-4a19-a660-ca681afd5ce8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f683 8d17ff97-bf7e-4ca4-a9a7-803f94b44843 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f683 8d17ff97-bf7e-4ca4-a9a7-803f94b44843 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735676227608,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d17ff97-bf7e-4ca4-a9a7-803f94b44843 {"id":"8d17ff97-bf7e-4ca4-a9a7-803f94b44843","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754338} /tidb/cdc/default/__cdc_meta__/capture/f357a362-09a5-4a19-a660-ca681afd5ce8 {"id":"f357a362-09a5-4a19-a660-ca681afd5ce8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f683 8d17ff97-bf7e-4ca4-a9a7-803f94b44843 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f683 8d17ff97-bf7e-4ca4-a9a7-803f94b44843 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test + grep -q 'failed to get info:' {"checkpoint-ts":457114735676227608,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d17ff97-bf7e-4ca4-a9a7-803f94b44843 {"id":"8d17ff97-bf7e-4ca4-a9a7-803f94b44843","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754338} /tidb/cdc/default/__cdc_meta__/capture/f357a362-09a5-4a19-a660-ca681afd5ce8 {"id":"f357a362-09a5-4a19-a660-ca681afd5ce8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754335} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f544 6c51d653-7e95-412e-8ce6-5e25a6b25c3f /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f683 8d17ff97-bf7e-4ca4-a9a7-803f94b44843 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2f683 8d17ff97-bf7e-4ca4-a9a7-803f94b44843 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735676227608,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 91000 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully check diff failed 2-th time, retry later check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseA-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:22 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a {"id":"8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754339} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3120498355177252900,"high":13612266207749673379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367775215902593,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:13.986016533+08:00","start-ts":457114734879834113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114736125018117} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114736636198919,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a {"id":"8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754339} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3120498355177252900,"high":13612266207749673379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367775215902593,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:13.986016533+08:00","start-ts":457114734879834113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114736125018117} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114736636198919,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a {"id":"8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754339} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3120498355177252900,"high":13612266207749673379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367775215902593,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:13.986016533+08:00","start-ts":457114734879834113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114736125018117} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114736636198919,"progress":1}' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1818618188.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseH-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_H/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_H/stdout0-2.log /tmp/tidb_cdc_test/fail_over_ddl_H/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_H/stdout1-1.log no DATA RACE found [Fri Apr 4 16:12:23 CST 2025] <<<<<< run test case fail_over_ddl_H success! >>>>>> command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 3-th time, retry later + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 5500 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_E case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_E.1917819180.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_E/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_E/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_N Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 3-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (30469) - No such process wait process 30469 exit for 1-th time... process 30469 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (30546) - No such process wait process 30546 exit for 1-th time... process 30546 already exit [Fri Apr 4 16:12:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.3072030722.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully check diff failed 4-th time, retry later [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c053e846-d542-4a1c-8200-d47ddddbeb67 {"id":"c053e846-d542-4a1c-8200-d47ddddbeb67","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734062469129,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c053e846-d542-4a1c-8200-d47ddddbeb67 {"id":"c053e846-d542-4a1c-8200-d47ddddbeb67","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734062469129,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c053e846-d542-4a1c-8200-d47ddddbeb67 {"id":"c053e846-d542-4a1c-8200-d47ddddbeb67","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734062469129,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:25 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.3078730789.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1409d809-6405-47ec-9971-cf2a2dabaed0 {"id":"1409d809-6405-47ec-9971-cf2a2dabaed0","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/capture/8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a {"id":"8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754339} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3120498355177252900,"high":13612266207749673379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367775215902593,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:13.986016533+08:00","start-ts":457114734879834113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114736125018117} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114738589171713,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1409d809-6405-47ec-9971-cf2a2dabaed0 {"id":"1409d809-6405-47ec-9971-cf2a2dabaed0","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/capture/8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a {"id":"8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754339} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3120498355177252900,"high":13612266207749673379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367775215902593,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:13.986016533+08:00","start-ts":457114734879834113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114736125018117} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114738589171713,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1409d809-6405-47ec-9971-cf2a2dabaed0 {"id":"1409d809-6405-47ec-9971-cf2a2dabaed0","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/capture/8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a {"id":"8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754339} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3ae4 133e6d36-747b-4a91-8b35-4cf69f53c198 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3bea 8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3120498355177252900,"high":13612266207749673379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367775215902593,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:13.986016533+08:00","start-ts":457114734879834113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114736125018117} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114738589171713,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.18258.out cli capture list + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:26 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040778c9-0001-42ce-bc0c-36cf3cac6f8e {"id":"040778c9-0001-42ce-bc0c-36cf3cac6f8e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754307} /tidb/cdc/default/__cdc_meta__/capture/9e7e261c-5131-4fb7-8329-c085d071bf9c {"id":"9e7e261c-5131-4fb7-8329-c085d071bf9c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754343} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac3177b 9e7e261c-5131-4fb7-8329-c085d071bf9c /tidb/cdc/default/__cdc_meta__/owner/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/__cdc_meta__/owner/223195ffdac3177b 9e7e261c-5131-4fb7-8329-c085d071bf9c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5136872072753862960,"high":17034506343219206841},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367615344892608,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:39.674226645+08:00","start-ts":457114725881741313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727126925319} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114736406822915,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040778c9-0001-42ce-bc0c-36cf3cac6f8e {"id":"040778c9-0001-42ce-bc0c-36cf3cac6f8e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754307} /tidb/cdc/default/__cdc_meta__/capture/9e7e261c-5131-4fb7-8329-c085d071bf9c {"id":"9e7e261c-5131-4fb7-8329-c085d071bf9c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754343} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac3177b 9e7e261c-5131-4fb7-8329-c085d071bf9c /tidb/cdc/default/__cdc_meta__/owner/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/__cdc_meta__/owner/223195ffdac3177b 9e7e261c-5131-4fb7-8329-c085d071bf9c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5136872072753862960,"high":17034506343219206841},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367615344892608,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:39.674226645+08:00","start-ts":457114725881741313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727126925319} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114736406822915,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040778c9-0001-42ce-bc0c-36cf3cac6f8e {"id":"040778c9-0001-42ce-bc0c-36cf3cac6f8e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754307} /tidb/cdc/default/__cdc_meta__/capture/9e7e261c-5131-4fb7-8329-c085d071bf9c {"id":"9e7e261c-5131-4fb7-8329-c085d071bf9c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754343} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdac3177b 9e7e261c-5131-4fb7-8329-c085d071bf9c /tidb/cdc/default/__cdc_meta__/owner/223195ffdac3112f 040778c9-0001-42ce-bc0c-36cf3cac6f8e /tidb/cdc/default/__cdc_meta__/owner/223195ffdac3177b 9e7e261c-5131-4fb7-8329-c085d071bf9c /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":5136872072753862960,"high":17034506343219206841},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367615344892608,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:39.674226645+08:00","start-ts":457114725881741313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114727126925319} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114736406822915,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully check diff failed 1-th time, retry later check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseL-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + ans='[ { "id": "8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "1409d809-6405-47ec-9971-cf2a2dabaed0", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"8e4c1ad3-5a3c-45f0-9a51-7d17804c8d3a",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"1409d809-6405-47ec-9971-cf2a2dabaed0",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=1409d809-6405-47ec-9971-cf2a2dabaed0 + '[' -z 1409d809-6405-47ec-9971-cf2a2dabaed0 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 1409d809-6405-47ec-9971-cf2a2dabaed0 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.18288.out cli changefeed move-table -c test -t 114 -d 1409d809-6405-47ec-9971-cf2a2dabaed0 check diff failed 5-th time, retry later [2025/04/04 16:12:27.749 +08:00] [DEBUG] [pd.rs:582] ["update latency statistics for raftstore-v1"] [tick=604] [thread_id=88] [2025/04/04 16:12:27.749 +08:00] [DEBUG] [pool.rs:146] ["scheduling task inspect raftstore latency: RaftDisk"] [thread_id=88] [2025/04/04 16:12:27.749 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="inspect raftstore latency: RaftDisk"] [thread_id=30] [2025/04/04 16:12:27.749 +08:00] [DEBUG] [pool.rs:146] ["scheduling task compute slow score: id 605, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(20.541ยตs), store_process_duration: Some(7.194ยตs), store_write_duration: Some(16.749ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=80] [2025/04/04 16:12:27.749 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="compute slow score: id 605, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(20.541ยตs), store_process_duration: Some(7.194ยตs), store_write_duration: Some(16.749ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=30] check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c053e846-d542-4a1c-8200-d47ddddbeb67 {"id":"c053e846-d542-4a1c-8200-d47ddddbeb67","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/capture/f4901844-2295-4560-977f-a0d0988255f7 {"id":"f4901844-2295-4560-977f-a0d0988255f7","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754345} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf452da f4901844-2295-4560-977f-a0d0988255f7 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf452da f4901844-2295-4560-977f-a0d0988255f7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734062469129,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c053e846-d542-4a1c-8200-d47ddddbeb67 {"id":"c053e846-d542-4a1c-8200-d47ddddbeb67","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/capture/f4901844-2295-4560-977f-a0d0988255f7 {"id":"f4901844-2295-4560-977f-a0d0988255f7","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754345} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf452da f4901844-2295-4560-977f-a0d0988255f7 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf452da f4901844-2295-4560-977f-a0d0988255f7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734062469129,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c053e846-d542-4a1c-8200-d47ddddbeb67 {"id":"c053e846-d542-4a1c-8200-d47ddddbeb67","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/capture/f4901844-2295-4560-977f-a0d0988255f7 {"id":"f4901844-2295-4560-977f-a0d0988255f7","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754345} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf44dc3 bdfd02be-7cf5-4e43-a6cc-23df5ba231c4 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf452da f4901844-2295-4560-977f-a0d0988255f7 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf45277 c053e846-d542-4a1c-8200-d47ddddbeb67 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf452da f4901844-2295-4560-977f-a0d0988255f7 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":7515052530135819116,"high":14478894183766776457},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367674212210317,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:51.736198826+08:00","start-ts":457114729042673666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114730287595528} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114734062469129,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore check diff failed 6-th time, retry later Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18761) - No such process wait process 18761 exit for 1-th time... process 18761 already exit [Fri Apr 4 16:12:28 CST 2025] <<<<<< START cdc server in fail_over_ddl_J case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_J.1897318975.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_J/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_J/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16189) - No such process wait process 16189 exit for 1-th time... process 16189 already exit [Fri Apr 4 16:12:27 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1636216364.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:30 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4dd559f8-3be4-45ae-a286-ccc8db29c335 {"id":"4dd559f8-3be4-45ae-a286-ccc8db29c335","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754348} /tidb/cdc/default/__cdc_meta__/capture/e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 {"id":"e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb87e 4dd559f8-3be4-45ae-a286-ccc8db29c335 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb87e 4dd559f8-3be4-45ae-a286-ccc8db29c335 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8811465680906879878,"high":13686026516719565716},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367703208224136,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:57.700031741+08:00","start-ts":457114730599546881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114731857838084} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735632711681,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4dd559f8-3be4-45ae-a286-ccc8db29c335 {"id":"4dd559f8-3be4-45ae-a286-ccc8db29c335","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754348} /tidb/cdc/default/__cdc_meta__/capture/e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 {"id":"e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb87e 4dd559f8-3be4-45ae-a286-ccc8db29c335 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb87e 4dd559f8-3be4-45ae-a286-ccc8db29c335 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8811465680906879878,"high":13686026516719565716},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367703208224136,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:57.700031741+08:00","start-ts":457114730599546881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114731857838084} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735632711681,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4dd559f8-3be4-45ae-a286-ccc8db29c335 {"id":"4dd559f8-3be4-45ae-a286-ccc8db29c335","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754348} /tidb/cdc/default/__cdc_meta__/capture/e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 {"id":"e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754322} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb0bb87e 4dd559f8-3be4-45ae-a286-ccc8db29c335 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb3c4 e3d1b840-53fd-4f31-a16e-1ccb0fe6bc60 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb0bb87e 4dd559f8-3be4-45ae-a286-ccc8db29c335 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8811465680906879878,"high":13686026516719565716},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367703208224136,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:57.700031741+08:00","start-ts":457114730599546881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114731857838084} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114735632711681,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseE-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_E/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_E/stdout0-2.log /tmp/tidb_cdc_test/fail_over_ddl_E/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_E/stdout1-1.log no DATA RACE found [Fri Apr 4 16:12:31 CST 2025] <<<<<< run test case fail_over_ddl_E success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0bacbca-30df-45a7-b4ed-4bd5eca70f74 {"id":"e0bacbca-30df-45a7-b4ed-4bd5eca70f74","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754348} /tidb/cdc/default/__cdc_meta__/capture/e2182464-08bf-417a-b0b0-d7ec30a70767 {"id":"e2182464-08bf-417a-b0b0-d7ec30a70767","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754323} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb123852 e0bacbca-30df-45a7-b4ed-4bd5eca70f74 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb123852 e0bacbca-30df-45a7-b4ed-4bd5eca70f74 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4128942869739245599,"high":4146714913998868127},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367709560548565,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.789451075+08:00","start-ts":457114730904682497,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732136759304} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114739345719303,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0bacbca-30df-45a7-b4ed-4bd5eca70f74 {"id":"e0bacbca-30df-45a7-b4ed-4bd5eca70f74","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754348} /tidb/cdc/default/__cdc_meta__/capture/e2182464-08bf-417a-b0b0-d7ec30a70767 {"id":"e2182464-08bf-417a-b0b0-d7ec30a70767","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754323} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb123852 e0bacbca-30df-45a7-b4ed-4bd5eca70f74 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb123852 e0bacbca-30df-45a7-b4ed-4bd5eca70f74 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4128942869739245599,"high":4146714913998868127},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367709560548565,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.789451075+08:00","start-ts":457114730904682497,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732136759304} /tidb/cdc/default/default/changefeed/status/test + grep -q 'failed to get info:' {"checkpoint-ts":457114739345719303,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0bacbca-30df-45a7-b4ed-4bd5eca70f74 {"id":"e0bacbca-30df-45a7-b4ed-4bd5eca70f74","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754348} /tidb/cdc/default/__cdc_meta__/capture/e2182464-08bf-417a-b0b0-d7ec30a70767 {"id":"e2182464-08bf-417a-b0b0-d7ec30a70767","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754323} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb123852 e0bacbca-30df-45a7-b4ed-4bd5eca70f74 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb1233c4 e2182464-08bf-417a-b0b0-d7ec30a70767 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb123852 e0bacbca-30df-45a7-b4ed-4bd5eca70f74 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":4128942869739245599,"high":4146714913998868127},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367709560548565,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:11:58.789451075+08:00","start-ts":457114730904682497,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114732136759304} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114739345719303,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 2-th time, retry later check diff failed 7-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_A Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18117) - No such process wait process 18117 exit for 1-th time... process 18117 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseM-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } check diff failed 8-th time, retry later [Pipeline] // withEnv VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367882734252854 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6f5c9c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:14074, start at 2025-04-04 16:12:32.46675289 +0800 CST m=+3.183133991 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:32.530 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:32.523 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:32.523 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367882734252854 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6f5c9c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:14074, start at 2025-04-04 16:12:32.46675289 +0800 CST m=+3.183133991 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:32.530 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:32.523 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:32.523 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367891089025294 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6f5fc80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:14153, start at 2025-04-04 16:12:32.633914268 +0800 CST m=+3.197380279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:32.641 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:32.626 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:32.626 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } check diff failed 2-th time, retry later [Pipeline] // stage [Pipeline] } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_L Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 check diff failed 9-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.15358.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later wait process cdc.test exit for 3-th time... check diff failed 10-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 91000 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + set +x + tso='457114741944352769 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114741944352769 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:12:37 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1539515397.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 4-th time... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 5-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 6-th time... process cdc.test already exit failOverCaseJ-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_J/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_J/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_J/stdout1-1.log /tmp/tidb_cdc_test/fail_over_ddl_J/stdout1-2.log no DATA RACE found [Fri Apr 4 16:12:38 CST 2025] <<<<<< run test case fail_over_ddl_J success! >>>>>> chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_M Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18757) - No such process wait process 18757 exit for 1-th time... process 18757 already exit [Fri Apr 4 16:12:36 CST 2025] <<<<<< START cdc server in fail_over_ddl_F case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeWrite=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_F.1894918951.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_F/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_F/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:39 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.1837718379.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49ab4092-c8ae-40bb-9609-d40b0f41aa7b {"id":"49ab4092-c8ae-40bb-9609-d40b0f41aa7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754328} /tidb/cdc/default/__cdc_meta__/capture/feb7a3e7-887c-4e77-89c3-0b376192c607 {"id":"feb7a3e7-887c-4e77-89c3-0b376192c607","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754357} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb291dd9 feb7a3e7-887c-4e77-89c3-0b376192c607 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/owner/223195ffdb291dd9 feb7a3e7-887c-4e77-89c3-0b376192c607 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15367819270581404014,"high":4849217813893219260},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367731149877193,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:04.327121447+08:00","start-ts":457114732336775169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114733594804229} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114737618714628,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49ab4092-c8ae-40bb-9609-d40b0f41aa7b {"id":"49ab4092-c8ae-40bb-9609-d40b0f41aa7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754328} /tidb/cdc/default/__cdc_meta__/capture/feb7a3e7-887c-4e77-89c3-0b376192c607 {"id":"feb7a3e7-887c-4e77-89c3-0b376192c607","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754357} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb291dd9 feb7a3e7-887c-4e77-89c3-0b376192c607 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/owner/223195ffdb291dd9 feb7a3e7-887c-4e77-89c3-0b376192c607 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15367819270581404014,"high":4849217813893219260},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367731149877193,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:04.327121447+08:00","start-ts":457114732336775169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114733594804229} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114737618714628,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49ab4092-c8ae-40bb-9609-d40b0f41aa7b {"id":"49ab4092-c8ae-40bb-9609-d40b0f41aa7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754328} /tidb/cdc/default/__cdc_meta__/capture/feb7a3e7-887c-4e77-89c3-0b376192c607 {"id":"feb7a3e7-887c-4e77-89c3-0b376192c607","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754357} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb291dd9 feb7a3e7-887c-4e77-89c3-0b376192c607 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb2918c3 49ab4092-c8ae-40bb-9609-d40b0f41aa7b /tidb/cdc/default/__cdc_meta__/owner/223195ffdb291dd9 feb7a3e7-887c-4e77-89c3-0b376192c607 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15367819270581404014,"high":4849217813893219260},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367731149877193,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:04.327121447+08:00","start-ts":457114732336775169,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114733594804229} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114737618714628,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 11-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:40 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/931b0e74-d99a-4268-9688-f8d3eef65406 {"id":"931b0e74-d99a-4268-9688-f8d3eef65406","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754357} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/931b0e74-d99a-4268-9688-f8d3eef65406 {"id":"931b0e74-d99a-4268-9688-f8d3eef65406","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754357} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/931b0e74-d99a-4268-9688-f8d3eef65406 {"id":"931b0e74-d99a-4268-9688-f8d3eef65406","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754357} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114741944352769 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.15464.out cli changefeed create --start-ts=457114741944352769 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367882734252854,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:12:40.941718138+08:00","start_ts":457114741944352769,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114741944352769,"checkpoint_ts":457114741944352769,"checkpoint_time":"2025-04-04 16:12:36.172","gid":{"low":2184079056153527260,"high":11614845187869143710}} PASS \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache coverage: 2.7% of statements in github.com/pingcap/ticdc/... [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage check diff failed 12-th time, retry later [Pipeline] } command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367925760934109 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6fde58000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:20230, start at 2025-04-04 16:12:40.747985071 +0800 CST m=+3.101311159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:40.755 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:40.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:40.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367925760934109 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6fde58000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:20230, start at 2025-04-04 16:12:40.747985071 +0800 CST m=+3.101311159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:40.755 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:40.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:40.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367918470980038 TiDB Cluster ID. tikv_gc_leader_uuid 657ff6fdcc80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-4l8h3--tr420, pid:20298, start at 2025-04-04 16:12:40.637698368 +0800 CST m=+2.872547542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:40.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:40.626 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:40.626 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_A/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:42 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1409d809-6405-47ec-9971-cf2a2dabaed0 {"id":"1409d809-6405-47ec-9971-cf2a2dabaed0","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/capture/b0fe94e2-6235-47f1-b741-578e43974920 {"id":"b0fe94e2-6235-47f1-b741-578e43974920","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754359} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3eff b0fe94e2-6235-47f1-b741-578e43974920 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3eff b0fe94e2-6235-47f1-b741-578e43974920 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3120498355177252900,"high":13612266207749673379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367775215902593,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:13.986016533+08:00","start-ts":457114734879834113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114736125018117} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114739375341571,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1409d809-6405-47ec-9971-cf2a2dabaed0 {"id":"1409d809-6405-47ec-9971-cf2a2dabaed0","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/capture/b0fe94e2-6235-47f1-b741-578e43974920 {"id":"b0fe94e2-6235-47f1-b741-578e43974920","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754359} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3eff b0fe94e2-6235-47f1-b741-578e43974920 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3eff b0fe94e2-6235-47f1-b741-578e43974920 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3120498355177252900,"high":13612266207749673379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367775215902593,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:13.986016533+08:00","start-ts":457114734879834113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114736125018117} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114739375341571,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1409d809-6405-47ec-9971-cf2a2dabaed0 {"id":"1409d809-6405-47ec-9971-cf2a2dabaed0","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754342} /tidb/cdc/default/__cdc_meta__/capture/b0fe94e2-6235-47f1-b741-578e43974920 {"id":"b0fe94e2-6235-47f1-b741-578e43974920","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754359} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdb4d3eff b0fe94e2-6235-47f1-b741-578e43974920 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3c4e 1409d809-6405-47ec-9971-cf2a2dabaed0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdb4d3eff b0fe94e2-6235-47f1-b741-578e43974920 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":3120498355177252900,"high":13612266207749673379},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367775215902593,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:13.986016533+08:00","start-ts":457114734879834113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114736125018117} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114739375341571,"progress":1}' + grep -q 'etcd info' + break + set +x /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16694) - No such process wait process 16694 exit for 1-th time... process 16694 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (16765) - No such process wait process 16765 exit for 1-th time... process 16765 already exit [Fri Apr 4 16:12:41 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1690016902.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully check diff failed 1-th time, retry later check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.cli.21527.out cli tso query --pd=http://127.0.0.1:2379 table fail_over_ddl_test.test2 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:44 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84104027-f8d2-452c-a8b7-5c9c71783cf5 {"id":"84104027-f8d2-452c-a8b7-5c9c71783cf5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754361} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114742242934785,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84104027-f8d2-452c-a8b7-5c9c71783cf5 {"id":"84104027-f8d2-452c-a8b7-5c9c71783cf5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754361} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114742242934785,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84104027-f8d2-452c-a8b7-5c9c71783cf5 {"id":"84104027-f8d2-452c-a8b7-5c9c71783cf5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754361} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114742242934785,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:44 CST 2025] <<<<<< START cdc server in fail_over_ddl_K case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_K.1697116973.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_K/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_K/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367931676216211 TiDB Cluster ID. tikv_gc_leader_uuid 657ff700124000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:17472, start at 2025-04-04 16:12:42.967347182 +0800 CST m=+3.113188183 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:42.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:43.003 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:43.003 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367931676216211 TiDB Cluster ID. tikv_gc_leader_uuid 657ff700124000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:17472, start at 2025-04-04 16:12:42.967347182 +0800 CST m=+3.113188183 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:42.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:43.003 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:43.003 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367929842517055 TiDB Cluster ID. tikv_gc_leader_uuid 657ff7002a8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-hbpq7--j195l, pid:17541, start at 2025-04-04 16:12:43.07046642 +0800 CST m=+3.104010302 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:43.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:43.100 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:43.100 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_L/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 13-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15400) - No such process wait process 15400 exit for 1-th time... process 15400 already exit [Fri Apr 4 16:12:45 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1553315535.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later + set +x + tso='457114744141643777 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114744141643777 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:12:45 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.2156321565.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later check diff failed 14-th time, retry later check diff failed 2-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.18833.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57d83129-c009-4e64-be52-ae14422d0eb5 {"id":"57d83129-c009-4e64-be52-ae14422d0eb5","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754364} /tidb/cdc/default/__cdc_meta__/capture/84104027-f8d2-452c-a8b7-5c9c71783cf5 {"id":"84104027-f8d2-452c-a8b7-5c9c71783cf5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754361} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2fb39 57d83129-c009-4e64-be52-ae14422d0eb5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2fb39 57d83129-c009-4e64-be52-ae14422d0eb5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114742242934785,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57d83129-c009-4e64-be52-ae14422d0eb5 {"id":"57d83129-c009-4e64-be52-ae14422d0eb5","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754364} /tidb/cdc/default/__cdc_meta__/capture/84104027-f8d2-452c-a8b7-5c9c71783cf5 {"id":"84104027-f8d2-452c-a8b7-5c9c71783cf5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754361} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2fb39 57d83129-c009-4e64-be52-ae14422d0eb5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2fb39 57d83129-c009-4e64-be52-ae14422d0eb5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114742242934785,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57d83129-c009-4e64-be52-ae14422d0eb5 {"id":"57d83129-c009-4e64-be52-ae14422d0eb5","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754364} /tidb/cdc/default/__cdc_meta__/capture/84104027-f8d2-452c-a8b7-5c9c71783cf5 {"id":"84104027-f8d2-452c-a8b7-5c9c71783cf5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754361} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2f622 f357a362-09a5-4a19-a660-ca681afd5ce8 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdaf2fb39 57d83129-c009-4e64-be52-ae14422d0eb5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2fad4 84104027-f8d2-452c-a8b7-5c9c71783cf5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdaf2fb39 57d83129-c009-4e64-be52-ae14422d0eb5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":1751658282563072813,"high":7566144674979320471},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367668127296815,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:10.790597342+08:00","start-ts":457114734037827586,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114735283273736} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114742242934785,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:48 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f26fa989-b8a3-4701-bb87-46a25d80cd2d {"id":"f26fa989-b8a3-4701-bb87-46a25d80cd2d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754365} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2184079056153527260,"high":11614845187869143710},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367882734252854,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:40.941718138+08:00","start-ts":457114741944352769,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114743189536775} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114743569645598,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f26fa989-b8a3-4701-bb87-46a25d80cd2d {"id":"f26fa989-b8a3-4701-bb87-46a25d80cd2d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754365} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2184079056153527260,"high":11614845187869143710},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367882734252854,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:40.941718138+08:00","start-ts":457114741944352769,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114743189536775} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114743569645598,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f26fa989-b8a3-4701-bb87-46a25d80cd2d {"id":"f26fa989-b8a3-4701-bb87-46a25d80cd2d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754365} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2184079056153527260,"high":11614845187869143710},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367882734252854,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:40.941718138+08:00","start-ts":457114741944352769,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114743189536775} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114743569645598,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:12:48 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=1*sleep(30000)->return(false)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1562015622.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseO-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory check diff failed 15-th time, retry later + set +x + tso='457114744731205633 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114744731205633 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:12:48 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1887018872.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseB-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [2025/04/04 16:12:48.767 +08:00] [DEBUG] [pool.rs:146] ["scheduling task heartbeat for region id: 106 start_key: 7480000000000000FF5200000000000000F8 end_key: 7480000000000000FF5400000000000000F8 region_epoch { conf_ver: 1 version: 49 } peers { id: 107 store_id: 1 }, leader 107, replication status None"] [thread_id=82] [2025/04/04 16:12:48.767 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="heartbeat for region id: 106 start_key: 7480000000000000FF5200000000000000F8 end_key: 7480000000000000FF5400000000000000F8 region_epoch { conf_ver: 1 version: 49 } peers { id: 107 store_id: 1 }, leader 107, replication status None"] [thread_id=30] [2025/04/04 16:12:48.767 +08:00] [DEBUG] [util.rs:486] ["request sent: 1"] [thread_id=30] Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367952783173265 TiDB Cluster ID. tikv_gc_leader_uuid 657ff704680000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:31791, start at 2025-04-04 16:12:47.41083299 +0800 CST m=+3.021752477 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:47.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:47.392 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:47.392 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367952783173265 TiDB Cluster ID. tikv_gc_leader_uuid 657ff704680000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:31791, start at 2025-04-04 16:12:47.41083299 +0800 CST m=+3.021752477 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:47.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:47.392 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:47.392 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489367955227907446 TiDB Cluster ID. tikv_gc_leader_uuid 657ff704988000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-fkq8d--j8bxs, pid:31869, start at 2025-04-04 16:12:47.623361747 +0800 CST m=+3.115762705 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:14:47.685 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:12:47.686 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:02:47.686 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_M/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:48 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39710428-bb22-4c10-8096-9113803e047a {"id":"39710428-bb22-4c10-8096-9113803e047a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754366} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39710428-bb22-4c10-8096-9113803e047a {"id":"39710428-bb22-4c10-8096-9113803e047a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754366} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39710428-bb22-4c10-8096-9113803e047a {"id":"39710428-bb22-4c10-8096-9113803e047a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754366} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114744141643777 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.cli.21626.out cli changefeed create --start-ts=457114744141643777 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367925760934109,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:12:49.334174937+08:00","start_ts":457114744141643777,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114744141643777,"checkpoint_ts":457114744141643777,"checkpoint_time":"2025-04-04 16:12:44.554","gid":{"low":9387995264054402309,"high":11749104953936480901}} PASS [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } coverage: 2.7% of statements in github.com/pingcap/ticdc/... [Pipeline] // stage [Pipeline] } check diff failed 2-th time, retry later [2025/04/04 16:12:49.845 +08:00] [DEBUG] [pd.rs:582] ["update latency statistics for raftstore-v1"] [tick=655] [thread_id=88] [2025/04/04 16:12:49.845 +08:00] [DEBUG] [pool.rs:146] ["scheduling task inspect raftstore latency: RaftDisk"] [thread_id=88] [2025/04/04 16:12:49.845 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="inspect raftstore latency: RaftDisk"] [thread_id=30] [2025/04/04 16:12:49.845 +08:00] [DEBUG] [pool.rs:146] ["scheduling task compute slow score: id 656, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(14.257ยตs), store_process_duration: Some(4.694ยตs), store_write_duration: Some(36.892ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=80] [2025/04/04 16:12:49.846 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="compute slow score: id 656, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(14.257ยตs), store_process_duration: Some(4.694ยตs), store_write_duration: Some(36.892ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=30] check diff failed 16-th time, retry later + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.33152.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:51 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/705d06f3-2043-4af0-a276-97cccb981d6a {"id":"705d06f3-2043-4af0-a276-97cccb981d6a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754368} /tidb/cdc/default/__cdc_meta__/capture/f26fa989-b8a3-4701-bb87-46a25d80cd2d {"id":"f26fa989-b8a3-4701-bb87-46a25d80cd2d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754365} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2184079056153527260,"high":11614845187869143710},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367882734252854,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:40.941718138+08:00","start-ts":457114741944352769,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114743189536775} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114745679642633,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/705d06f3-2043-4af0-a276-97cccb981d6a {"id":"705d06f3-2043-4af0-a276-97cccb981d6a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754368} /tidb/cdc/default/__cdc_meta__/capture/f26fa989-b8a3-4701-bb87-46a25d80cd2d {"id":"f26fa989-b8a3-4701-bb87-46a25d80cd2d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754365} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2184079056153527260,"high":11614845187869143710},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367882734252854,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:40.941718138+08:00","start-ts":457114741944352769,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114743189536775} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114745679642633,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/705d06f3-2043-4af0-a276-97cccb981d6a {"id":"705d06f3-2043-4af0-a276-97cccb981d6a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754368} /tidb/cdc/default/__cdc_meta__/capture/f26fa989-b8a3-4701-bb87-46a25d80cd2d {"id":"f26fa989-b8a3-4701-bb87-46a25d80cd2d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754365} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4bfed 931b0e74-d99a-4268-9688-f8d3eef65406 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c0c8 f26fa989-b8a3-4701-bb87-46a25d80cd2d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2184079056153527260,"high":11614845187869143710},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367882734252854,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:40.941718138+08:00","start-ts":457114741944352769,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114743189536775} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114745679642633,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.15702.out cli capture list + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:51 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/896729a7-2686-4207-bafa-e6c4d0b3bb39 {"id":"896729a7-2686-4207-bafa-e6c4d0b3bb39","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754368} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/896729a7-2686-4207-bafa-e6c4d0b3bb39 {"id":"896729a7-2686-4207-bafa-e6c4d0b3bb39","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754368} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/896729a7-2686-4207-bafa-e6c4d0b3bb39 {"id":"896729a7-2686-4207-bafa-e6c4d0b3bb39","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754368} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114744731205633 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.18928.out cli changefeed create --start-ts=457114744731205633 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489367931676216211,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:12:51.57902021+08:00","start_ts":457114744731205633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114744731205633,"checkpoint_ts":457114744731205633,"checkpoint_time":"2025-04-04 16:12:46.803","gid":{"low":13423124298916295285,"high":16500626430214257564}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseK-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_K/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_K/stdout0-2.log /tmp/tidb_cdc_test/fail_over_ddl_K/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_K/stdout1-1.log /tmp/tidb_cdc_test/fail_over_ddl_K/stdout1-2.log no DATA RACE found [Fri Apr 4 16:12:52 CST 2025] <<<<<< run test case fail_over_ddl_K success! >>>>>> check diff failed 17-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + tso='457114745920815105 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114745920815105 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:12:52 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.3319033192.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + ans='[ { "id": "705d06f3-2043-4af0-a276-97cccb981d6a", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" }, { "id": "f26fa989-b8a3-4701-bb87-46a25d80cd2d", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"705d06f3-2043-4af0-a276-97cccb981d6a",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '},' '{' '"id":' '"f26fa989-b8a3-4701-bb87-46a25d80cd2d",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=705d06f3-2043-4af0-a276-97cccb981d6a + '[' -z 705d06f3-2043-4af0-a276-97cccb981d6a ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 705d06f3-2043-4af0-a276-97cccb981d6a + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.15729.out cli changefeed move-table -c test -t 114 -d 705d06f3-2043-4af0-a276-97cccb981d6a command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 4-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (21568) - No such process wait process 21568 exit for 1-th time... process 21568 already exit [Fri Apr 4 16:12:53 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.2170021702.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 not exists for 1-th check, retry later check diff failed 18-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_O Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:55 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34fef8b0-2926-4518-b1e4-81d2ffd7688d {"id":"34fef8b0-2926-4518-b1e4-81d2ffd7688d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754373} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34fef8b0-2926-4518-b1e4-81d2ffd7688d {"id":"34fef8b0-2926-4518-b1e4-81d2ffd7688d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754373} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34fef8b0-2926-4518-b1e4-81d2ffd7688d {"id":"34fef8b0-2926-4518-b1e4-81d2ffd7688d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754373} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114745920815105 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.33252.out cli changefeed create --start-ts=457114745920815105 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 Create changefeed successfully! ID: test Info: {"upstream_id":7489367952783173265,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:12:56.100704032+08:00","start_ts":457114745920815105,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114745920815105,"checkpoint_ts":457114745920815105,"checkpoint_time":"2025-04-04 16:12:51.341","gid":{"low":15438872866450531533,"high":2050275118990919340}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/554e261d-539b-45e2-b0af-a165df5af139 {"id":"554e261d-539b-45e2-b0af-a165df5af139","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754373} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114745766936616,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/554e261d-539b-45e2-b0af-a165df5af139 {"id":"554e261d-539b-45e2-b0af-a165df5af139","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754373} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114745766936616,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/554e261d-539b-45e2-b0af-a165df5af139 {"id":"554e261d-539b-45e2-b0af-a165df5af139","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754373} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114745766936616,"progress":1}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 16:12:56 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS=github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.2177721779.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 exists check diff failed 19-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_B Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18875) - No such process wait process 18875 exit for 1-th time... process 18875 already exit [Fri Apr 4 16:12:57 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1901519017.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 20-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:12:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/554e261d-539b-45e2-b0af-a165df5af139 {"id":"554e261d-539b-45e2-b0af-a165df5af139","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754373} /tidb/cdc/default/__cdc_meta__/capture/61c49857-a4e3-42a4-9530-62bb3318878f {"id":"61c49857-a4e3-42a4-9530-62bb3318878f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754376} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38c26 61c49857-a4e3-42a4-9530-62bb3318878f /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38c26 61c49857-a4e3-42a4-9530-62bb3318878f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114745766936616,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/554e261d-539b-45e2-b0af-a165df5af139 {"id":"554e261d-539b-45e2-b0af-a165df5af139","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754373} /tidb/cdc/default/__cdc_meta__/capture/61c49857-a4e3-42a4-9530-62bb3318878f {"id":"61c49857-a4e3-42a4-9530-62bb3318878f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754376} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38c26 61c49857-a4e3-42a4-9530-62bb3318878f /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38c26 61c49857-a4e3-42a4-9530-62bb3318878f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114745766936616,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/554e261d-539b-45e2-b0af-a165df5af139 {"id":"554e261d-539b-45e2-b0af-a165df5af139","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754373} /tidb/cdc/default/__cdc_meta__/capture/61c49857-a4e3-42a4-9530-62bb3318878f {"id":"61c49857-a4e3-42a4-9530-62bb3318878f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754376} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38ae4 39710428-bb22-4c10-8096-9113803e047a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38c26 61c49857-a4e3-42a4-9530-62bb3318878f /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38c26 61c49857-a4e3-42a4-9530-62bb3318878f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114745766936616,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (33195) - No such process wait process 33195 exit for 1-th time... process 33195 already exit [Fri Apr 4 16:13:00 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.3332033322.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5013aeab-9077-46fd-ab59-d77813973bd6 {"id":"5013aeab-9077-46fd-ab59-d77813973bd6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754377} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13423124298916295285,"high":16500626430214257564},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367931676216211,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:51.57902021+08:00","start-ts":457114744731205633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745976389640} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114746487570447,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5013aeab-9077-46fd-ab59-d77813973bd6 {"id":"5013aeab-9077-46fd-ab59-d77813973bd6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754377} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13423124298916295285,"high":16500626430214257564},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367931676216211,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:51.57902021+08:00","start-ts":457114744731205633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745976389640} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114746487570447,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5013aeab-9077-46fd-ab59-d77813973bd6 {"id":"5013aeab-9077-46fd-ab59-d77813973bd6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754377} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13423124298916295285,"high":16500626430214257564},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367931676216211,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:51.57902021+08:00","start-ts":457114744731205633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745976389640} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114746487570447,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:13:00 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockAfterPass=pause' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1908719089.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseI-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m check diff failed 2-th time, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir [Pipeline] } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fe1f0a4-edd7-4949-95ef-22aa18d0f77d {"id":"5fe1f0a4-edd7-4949-95ef-22aa18d0f77d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754380} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114747546107917,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fe1f0a4-edd7-4949-95ef-22aa18d0f77d {"id":"5fe1f0a4-edd7-4949-95ef-22aa18d0f77d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754380} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114747546107917,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fe1f0a4-edd7-4949-95ef-22aa18d0f77d {"id":"5fe1f0a4-edd7-4949-95ef-22aa18d0f77d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754380} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114747546107917,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:13:03 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=pause' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.3340333405.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 16:13:03.192 +08:00] [DEBUG] [pd.rs:582] ["update latency statistics for raftstore-v1"] [tick=943] [thread_id=88] [2025/04/04 16:13:03.192 +08:00] [DEBUG] [pool.rs:146] ["scheduling task inspect raftstore latency: RaftDisk"] [thread_id=88] [2025/04/04 16:13:03.192 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="inspect raftstore latency: RaftDisk"] [thread_id=30] [2025/04/04 16:13:03.200 +08:00] [DEBUG] [pool.rs:146] ["scheduling task compute slow score: id 944, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(8.306427ms), store_process_duration: Some(9.036ยตs), store_write_duration: Some(30.668ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=80] [2025/04/04 16:13:03.200 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="compute slow score: id 944, factor: RaftDisk, duration RaftstoreDuration { store_wait_duration: Some(8.306427ms), store_process_duration: Some(9.036ยตs), store_write_duration: Some(30.668ยตs), store_commit_duration: Some(0ns), apply_wait_duration: None, apply_process_duration: None }"] [thread_id=30] command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3db28a55-65e3-4107-a54c-24938b78aeb8 {"id":"3db28a55-65e3-4107-a54c-24938b78aeb8","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754380} /tidb/cdc/default/__cdc_meta__/capture/5013aeab-9077-46fd-ab59-d77813973bd6 {"id":"5013aeab-9077-46fd-ab59-d77813973bd6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754377} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde164f 3db28a55-65e3-4107-a54c-24938b78aeb8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde164f 3db28a55-65e3-4107-a54c-24938b78aeb8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13423124298916295285,"high":16500626430214257564},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367931676216211,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:51.57902021+08:00","start-ts":457114744731205633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745976389640} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114746487570447,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3db28a55-65e3-4107-a54c-24938b78aeb8 {"id":"3db28a55-65e3-4107-a54c-24938b78aeb8","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754380} /tidb/cdc/default/__cdc_meta__/capture/5013aeab-9077-46fd-ab59-d77813973bd6 {"id":"5013aeab-9077-46fd-ab59-d77813973bd6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754377} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde164f 3db28a55-65e3-4107-a54c-24938b78aeb8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde164f 3db28a55-65e3-4107-a54c-24938b78aeb8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13423124298916295285,"high":16500626430214257564},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367931676216211,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:51.57902021+08:00","start-ts":457114744731205633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745976389640} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114746487570447,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3db28a55-65e3-4107-a54c-24938b78aeb8 {"id":"3db28a55-65e3-4107-a54c-24938b78aeb8","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754380} /tidb/cdc/default/__cdc_meta__/capture/5013aeab-9077-46fd-ab59-d77813973bd6 {"id":"5013aeab-9077-46fd-ab59-d77813973bd6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754377} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde14e5 896729a7-2686-4207-bafa-e6c4d0b3bb39 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde164f 3db28a55-65e3-4107-a54c-24938b78aeb8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde164f 3db28a55-65e3-4107-a54c-24938b78aeb8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13423124298916295285,"high":16500626430214257564},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367931676216211,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:51.57902021+08:00","start-ts":457114744731205633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745976389640} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114746487570447,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.19157.out cli capture list /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (21705) - No such process wait process 21705 exit for 1-th time... process 21705 already exit ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (21782) - No such process wait process 21782 exit for 1-th time... process 21782 already exit [Fri Apr 4 16:13:04 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.2191621918.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368023286308762 TiDB Cluster ID. tikv_gc_leader_uuid 657ff7144e00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:17408, start at 2025-04-04 16:13:03.680530838 +0800 CST m=+3.424982658 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:03.745 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:03.744 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:03.744 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368023286308762 TiDB Cluster ID. tikv_gc_leader_uuid 657ff7144e00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:17408, start at 2025-04-04 16:13:03.680530838 +0800 CST m=+3.424982658 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:03.745 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:03.744 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:03.744 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368019985657599 TiDB Cluster ID. tikv_gc_leader_uuid 657ff7144700006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-6mcfg--9wrqf, pid:17484, start at 2025-04-04 16:13:03.652015479 +0800 CST m=+3.282027598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:03.659 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:03.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:03.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_O/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseF-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_F/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_F/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_F/stdout1-1.log /tmp/tidb_cdc_test/fail_over_ddl_F/stdout1-2.log no DATA RACE found [Fri Apr 4 16:13:04 CST 2025] <<<<<< run test case fail_over_ddl_F success! >>>>>> + set +x + ans='[ { "id": "5013aeab-9077-46fd-ab59-d77813973bd6", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "3db28a55-65e3-4107-a54c-24938b78aeb8", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"5013aeab-9077-46fd-ab59-d77813973bd6",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"3db28a55-65e3-4107-a54c-24938b78aeb8",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=3db28a55-65e3-4107-a54c-24938b78aeb8 + '[' -z 3db28a55-65e3-4107-a54c-24938b78aeb8 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 3db28a55-65e3-4107-a54c-24938b78aeb8 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.cli.19184.out cli changefeed move-table -c test -t 114 -d 3db28a55-65e3-4107-a54c-24938b78aeb8 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fe1f0a4-edd7-4949-95ef-22aa18d0f77d {"id":"5fe1f0a4-edd7-4949-95ef-22aa18d0f77d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754380} /tidb/cdc/default/__cdc_meta__/capture/efa4f66b-da20-4c3e-bf13-effd119fafef {"id":"efa4f66b-da20-4c3e-bf13-effd119fafef","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754383} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c428 efa4f66b-da20-4c3e-bf13-effd119fafef /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c428 efa4f66b-da20-4c3e-bf13-effd119fafef /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114747546107917,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fe1f0a4-edd7-4949-95ef-22aa18d0f77d {"id":"5fe1f0a4-edd7-4949-95ef-22aa18d0f77d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754380} /tidb/cdc/default/__cdc_meta__/capture/efa4f66b-da20-4c3e-bf13-effd119fafef {"id":"efa4f66b-da20-4c3e-bf13-effd119fafef","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754383} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c428 efa4f66b-da20-4c3e-bf13-effd119fafef /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c428 efa4f66b-da20-4c3e-bf13-effd119fafef /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114747546107917,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fe1f0a4-edd7-4949-95ef-22aa18d0f77d {"id":"5fe1f0a4-edd7-4949-95ef-22aa18d0f77d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754380} /tidb/cdc/default/__cdc_meta__/capture/efa4f66b-da20-4c3e-bf13-effd119fafef {"id":"efa4f66b-da20-4c3e-bf13-effd119fafef","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754383} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c2e4 34fef8b0-2926-4518-b1e4-81d2ffd7688d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c428 efa4f66b-da20-4c3e-bf13-effd119fafef /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c428 efa4f66b-da20-4c3e-bf13-effd119fafef /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114747546107917,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.33469.out cli capture list + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.18734.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368032724137380 TiDB Cluster ID. tikv_gc_leader_uuid 657ff7169ac0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:19439, start at 2025-04-04 16:13:06.04237944 +0800 CST m=+3.451314502 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:06.120 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:06.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:06.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368032724137380 TiDB Cluster ID. tikv_gc_leader_uuid 657ff7169ac0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:19439, start at 2025-04-04 16:13:06.04237944 +0800 CST m=+3.451314502 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:06.120 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:06.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:06.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368023355407983 TiDB Cluster ID. tikv_gc_leader_uuid 657ff716b30000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-wvb9t--dbddj, pid:19520, start at 2025-04-04 16:13:06.146788067 +0800 CST m=+3.404833408 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:06.154 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:06.174 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:06.174 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_B/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x + ans='[ { "id": "5fe1f0a4-edd7-4949-95ef-22aa18d0f77d", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "efa4f66b-da20-4c3e-bf13-effd119fafef", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"5fe1f0a4-edd7-4949-95ef-22aa18d0f77d",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"efa4f66b-da20-4c3e-bf13-effd119fafef",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=efa4f66b-da20-4c3e-bf13-effd119fafef + '[' -z efa4f66b-da20-4c3e-bf13-effd119fafef ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d efa4f66b-da20-4c3e-bf13-effd119fafef + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.cli.33500.out cli changefeed move-table -c test -t 114 -d efa4f66b-da20-4c3e-bf13-effd119fafef + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:07 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/902222a1-8fd8-4b53-9951-627bbedf6289 {"id":"902222a1-8fd8-4b53-9951-627bbedf6289","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754385} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114748126494727,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/902222a1-8fd8-4b53-9951-627bbedf6289 {"id":"902222a1-8fd8-4b53-9951-627bbedf6289","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754385} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114748126494727,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/902222a1-8fd8-4b53-9951-627bbedf6289 {"id":"902222a1-8fd8-4b53-9951-627bbedf6289","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754385} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114748126494727,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:13:07 CST 2025] <<<<<< START cdc server in fail_over_ddl_A case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_A.2199822000.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_A/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_A/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114750070554625 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114750070554625 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:13:08 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1876018762.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_I Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.20731.out cli tso query --pd=http://127.0.0.1:2379 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x + tso='457114750688165889 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114750688165889 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:13:10 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.2077220774.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:10 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b33ebaa-e106-4c76-ac28-0317ff86510f {"id":"8b33ebaa-e106-4c76-ac28-0317ff86510f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754388} /tidb/cdc/default/__cdc_meta__/capture/902222a1-8fd8-4b53-9951-627bbedf6289 {"id":"902222a1-8fd8-4b53-9951-627bbedf6289","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754385} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38dfd 8b33ebaa-e106-4c76-ac28-0317ff86510f /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38dfd 8b33ebaa-e106-4c76-ac28-0317ff86510f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114748126494727,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b33ebaa-e106-4c76-ac28-0317ff86510f {"id":"8b33ebaa-e106-4c76-ac28-0317ff86510f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754388} /tidb/cdc/default/__cdc_meta__/capture/902222a1-8fd8-4b53-9951-627bbedf6289 {"id":"902222a1-8fd8-4b53-9951-627bbedf6289","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754385} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38dfd 8b33ebaa-e106-4c76-ac28-0317ff86510f /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38dfd 8b33ebaa-e106-4c76-ac28-0317ff86510f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114748126494727,"progress":1}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b33ebaa-e106-4c76-ac28-0317ff86510f {"id":"8b33ebaa-e106-4c76-ac28-0317ff86510f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754388} /tidb/cdc/default/__cdc_meta__/capture/902222a1-8fd8-4b53-9951-627bbedf6289 {"id":"902222a1-8fd8-4b53-9951-627bbedf6289","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754385} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38bc1 554e261d-539b-45e2-b0af-a165df5af139 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbd38dfd 8b33ebaa-e106-4c76-ac28-0317ff86510f /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38d96 902222a1-8fd8-4b53-9951-627bbedf6289 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbd38dfd 8b33ebaa-e106-4c76-ac28-0317ff86510f /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9387995264054402309,"high":11749104953936480901},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367925760934109,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:49.334174937+08:00","start-ts":457114744141643777,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745386827783} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114748126494727,"progress":1}' + break + set +x check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:11 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac0f691b-d671-4f05-a89c-542f6e25be91 {"id":"ac0f691b-d671-4f05-a89c-542f6e25be91","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754388} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac0f691b-d671-4f05-a89c-542f6e25be91 {"id":"ac0f691b-d671-4f05-a89c-542f6e25be91","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754388} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac0f691b-d671-4f05-a89c-542f6e25be91 {"id":"ac0f691b-d671-4f05-a89c-542f6e25be91","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754388} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114750070554625 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.18825.out cli changefeed create --start-ts=457114750070554625 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489368023286308762,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:13:11.918771711+08:00","start_ts":457114750070554625,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114750070554625,"checkpoint_ts":457114750070554625,"checkpoint_time":"2025-04-04 16:13:07.171","gid":{"low":16087946732044607413,"high":17203305829823890572}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully check diff failed 2-th time, retry later + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (15538) - No such process wait process 15538 exit for 1-th time... process 15538 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 18200 0 --:--:-- --:--:-- --:--:-- 18200 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:13 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f961bede-2e6b-4539-9efb-2e840e70c3fb {"id":"f961bede-2e6b-4539-9efb-2e840e70c3fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754391} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb /tidb/cdc/default/__cdc_meta__/owner/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f961bede-2e6b-4539-9efb-2e840e70c3fb {"id":"f961bede-2e6b-4539-9efb-2e840e70c3fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754391} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb /tidb/cdc/default/__cdc_meta__/owner/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f961bede-2e6b-4539-9efb-2e840e70c3fb {"id":"f961bede-2e6b-4539-9efb-2e840e70c3fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754391} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb /tidb/cdc/default/__cdc_meta__/owner/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114750688165889 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.20832.out cli changefeed create --start-ts=457114750688165889 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489368032724137380,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:13:14.260802664+08:00","start_ts":457114750688165889,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114750688165889,"checkpoint_ts":457114750688165889,"checkpoint_time":"2025-04-04 16:13:09.527","gid":{"low":8088882069560617283,"high":5566475742481669810}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table fail_over_ddl_test.test2 exists + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 91000 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } + set +x Command succeeded on attempt 1 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 [Pipeline] // stage [Pipeline] } table fail_over_ddl_test.test2 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseA-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_A/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_A/stdout0-2.log /tmp/tidb_cdc_test/fail_over_ddl_A/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_A/stdout1-1.log /tmp/tidb_cdc_test/fail_over_ddl_A/stdout1-2.log no DATA RACE found [Fri Apr 4 16:13:15 CST 2025] <<<<<< run test case fail_over_ddl_A success! >>>>>> /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18765) - No such process wait process 18765 exit for 1-th time... process 18765 already exit [Fri Apr 4 16:13:15 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1890218904.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:13:17 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1585615858.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (20777) - No such process wait process 20777 exit for 1-th time... process 20777 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:13:18 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitReportAfterWrite=pause' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.2091120913.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368078723936793 TiDB Cluster ID. tikv_gc_leader_uuid 657ff721550000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:18081, start at 2025-04-04 16:13:17.03503267 +0800 CST m=+3.089982652 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:17.042 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:17.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:17.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368078723936793 TiDB Cluster ID. tikv_gc_leader_uuid 657ff721550000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:18081, start at 2025-04-04 16:13:17.03503267 +0800 CST m=+3.089982652 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:17.042 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:17.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:17.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368083987935385 TiDB Cluster ID. tikv_gc_leader_uuid 657ff721864000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-c2312--mpzgq, pid:18167, start at 2025-04-04 16:13:17.235641804 +0800 CST m=+3.123868589 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:17.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:17.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:17.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_I/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3087 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22fdf121-935b-42bc-889f-6ce21fc3d409 {"id":"22fdf121-935b-42bc-889f-6ce21fc3d409","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754396} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16087946732044607413,"high":17203305829823890572},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368023286308762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:11.918771711+08:00","start-ts":457114750070554625,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751316000770} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114751696109570,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22fdf121-935b-42bc-889f-6ce21fc3d409 {"id":"22fdf121-935b-42bc-889f-6ce21fc3d409","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754396} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16087946732044607413,"high":17203305829823890572},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368023286308762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:11.918771711+08:00","start-ts":457114750070554625,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751316000770} /tidb/cdc/default/default/changefeed/status/test + grep -q 'failed to get info:' {"checkpoint-ts":457114751696109570,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22fdf121-935b-42bc-889f-6ce21fc3d409 {"id":"22fdf121-935b-42bc-889f-6ce21fc3d409","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754396} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16087946732044607413,"high":17203305829823890572},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368023286308762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:11.918771711+08:00","start-ts":457114750070554625,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751316000770} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114751696109570,"progress":1}' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:13:19 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=sleep(30000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1898118983.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/049c0de9-a80a-4ab8-8ce0-f076d59ceae8 {"id":"049c0de9-a80a-4ab8-8ce0-f076d59ceae8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754397} /tidb/cdc/default/__cdc_meta__/capture/705d06f3-2043-4af0-a276-97cccb981d6a {"id":"705d06f3-2043-4af0-a276-97cccb981d6a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754368} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c627 049c0de9-a80a-4ab8-8ce0-f076d59ceae8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c627 049c0de9-a80a-4ab8-8ce0-f076d59ceae8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2184079056153527260,"high":11614845187869143710},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367882734252854,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:40.941718138+08:00","start-ts":457114741944352769,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114743189536775} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114746990624780,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/049c0de9-a80a-4ab8-8ce0-f076d59ceae8 {"id":"049c0de9-a80a-4ab8-8ce0-f076d59ceae8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754397} /tidb/cdc/default/__cdc_meta__/capture/705d06f3-2043-4af0-a276-97cccb981d6a {"id":"705d06f3-2043-4af0-a276-97cccb981d6a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754368} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c627 049c0de9-a80a-4ab8-8ce0-f076d59ceae8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c627 049c0de9-a80a-4ab8-8ce0-f076d59ceae8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2184079056153527260,"high":11614845187869143710},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367882734252854,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:40.941718138+08:00","start-ts":457114741944352769,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114743189536775} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114746990624780,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/049c0de9-a80a-4ab8-8ce0-f076d59ceae8 {"id":"049c0de9-a80a-4ab8-8ce0-f076d59ceae8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754397} /tidb/cdc/default/__cdc_meta__/capture/705d06f3-2043-4af0-a276-97cccb981d6a {"id":"705d06f3-2043-4af0-a276-97cccb981d6a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754368} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbb4c627 049c0de9-a80a-4ab8-8ce0-f076d59ceae8 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c12a 705d06f3-2043-4af0-a276-97cccb981d6a /tidb/cdc/default/__cdc_meta__/owner/223195ffdbb4c627 049c0de9-a80a-4ab8-8ce0-f076d59ceae8 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":2184079056153527260,"high":11614845187869143710},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367882734252854,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:40.941718138+08:00","start-ts":457114741944352769,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114743189536775} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114746990624780,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.19372.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/065c48ff-2202-499a-8dc8-e25a7aa43b31 {"id":"065c48ff-2202-499a-8dc8-e25a7aa43b31","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754398} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8088882069560617283,"high":5566475742481669810},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368032724137380,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:14.260802664+08:00","start-ts":457114750688165889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751920504842} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114752300613666,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/065c48ff-2202-499a-8dc8-e25a7aa43b31 {"id":"065c48ff-2202-499a-8dc8-e25a7aa43b31","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754398} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8088882069560617283,"high":5566475742481669810},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368032724137380,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:14.260802664+08:00","start-ts":457114750688165889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751920504842} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114752300613666,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/065c48ff-2202-499a-8dc8-e25a7aa43b31 {"id":"065c48ff-2202-499a-8dc8-e25a7aa43b31","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754398} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8088882069560617283,"high":5566475742481669810},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368032724137380,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:14.260802664+08:00","start-ts":457114750688165889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751920504842} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114752300613666,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:13:21 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.2099220994.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later + set +x + tso='457114753672675329 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114753672675329 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:13:22 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1940319405.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:22 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3668 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22fdf121-935b-42bc-889f-6ce21fc3d409 {"id":"22fdf121-935b-42bc-889f-6ce21fc3d409","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754396} /tidb/cdc/default/__cdc_meta__/capture/8a40ce3c-948b-44a6-9491-2200bd9e3656 {"id":"8a40ce3c-948b-44a6-9491-2200bd9e3656","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754399} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c224 8a40ce3c-948b-44a6-9491-2200bd9e3656 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c224 8a40ce3c-948b-44a6-9491-2200bd9e3656 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16087946732044607413,"high":17203305829823890572},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368023286308762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:11.918771711+08:00","start-ts":457114750070554625,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751316000770} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114751696109570,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22fdf121-935b-42bc-889f-6ce21fc3d409 {"id":"22fdf121-935b-42bc-889f-6ce21fc3d409","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754396} /tidb/cdc/default/__cdc_meta__/capture/8a40ce3c-948b-44a6-9491-2200bd9e3656 {"id":"8a40ce3c-948b-44a6-9491-2200bd9e3656","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754399} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c224 8a40ce3c-948b-44a6-9491-2200bd9e3656 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c224 8a40ce3c-948b-44a6-9491-2200bd9e3656 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16087946732044607413,"high":17203305829823890572},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368023286308762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:11.918771711+08:00","start-ts":457114750070554625,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751316000770} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114751696109570,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22fdf121-935b-42bc-889f-6ce21fc3d409 {"id":"22fdf121-935b-42bc-889f-6ce21fc3d409","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754396} /tidb/cdc/default/__cdc_meta__/capture/8a40ce3c-948b-44a6-9491-2200bd9e3656 {"id":"8a40ce3c-948b-44a6-9491-2200bd9e3656","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754399} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c0e3 ac0f691b-d671-4f05-a89c-542f6e25be91 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c224 8a40ce3c-948b-44a6-9491-2200bd9e3656 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c224 8a40ce3c-948b-44a6-9491-2200bd9e3656 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16087946732044607413,"high":17203305829823890572},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368023286308762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:11.918771711+08:00","start-ts":457114750070554625,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751316000770} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114751696109570,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.19058.out cli capture list + set +x + ans='[ { "id": "8a40ce3c-948b-44a6-9491-2200bd9e3656", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" }, { "id": "22fdf121-935b-42bc-889f-6ce21fc3d409", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"8a40ce3c-948b-44a6-9491-2200bd9e3656",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '},' '{' '"id":' '"22fdf121-935b-42bc-889f-6ce21fc3d409",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=8a40ce3c-948b-44a6-9491-2200bd9e3656 + '[' -z 8a40ce3c-948b-44a6-9491-2200bd9e3656 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 8a40ce3c-948b-44a6-9491-2200bd9e3656 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.cli.19086.out cli changefeed move-table -c test -t 114 -d 8a40ce3c-948b-44a6-9491-2200bd9e3656 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/065c48ff-2202-499a-8dc8-e25a7aa43b31 {"id":"065c48ff-2202-499a-8dc8-e25a7aa43b31","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754398} /tidb/cdc/default/__cdc_meta__/capture/47106095-61c9-4e83-944a-90ea4be3361e {"id":"47106095-61c9-4e83-944a-90ea4be3361e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754401} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8088882069560617283,"high":5566475742481669810},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368032724137380,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:14.260802664+08:00","start-ts":457114750688165889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751920504842} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114752300613666,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/065c48ff-2202-499a-8dc8-e25a7aa43b31 {"id":"065c48ff-2202-499a-8dc8-e25a7aa43b31","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754398} /tidb/cdc/default/__cdc_meta__/capture/47106095-61c9-4e83-944a-90ea4be3361e {"id":"47106095-61c9-4e83-944a-90ea4be3361e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754401} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8088882069560617283,"high":5566475742481669810},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368032724137380,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:14.260802664+08:00","start-ts":457114750688165889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751920504842} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114752300613666,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/065c48ff-2202-499a-8dc8-e25a7aa43b31 {"id":"065c48ff-2202-499a-8dc8-e25a7aa43b31","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754398} /tidb/cdc/default/__cdc_meta__/capture/47106095-61c9-4e83-944a-90ea4be3361e {"id":"47106095-61c9-4e83-944a-90ea4be3361e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754401} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3514e3 f961bede-2e6b-4539-9efb-2e840e70c3fb /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8088882069560617283,"high":5566475742481669810},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368032724137380,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:14.260802664+08:00","start-ts":457114750688165889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751920504842} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114752300613666,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.21059.out cli capture list + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:25 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe9647aa-9dbb-4a4e-a803-404c0cc46910 {"id":"fe9647aa-9dbb-4a4e-a803-404c0cc46910","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754402} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe9647aa-9dbb-4a4e-a803-404c0cc46910 {"id":"fe9647aa-9dbb-4a4e-a803-404c0cc46910","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754402} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe9647aa-9dbb-4a4e-a803-404c0cc46910 {"id":"fe9647aa-9dbb-4a4e-a803-404c0cc46910","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754402} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114753672675329 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.19464.out cli changefeed create --start-ts=457114753672675329 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489368078723936793,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:13:25.652264049+08:00","start_ts":457114753672675329,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114753672675329,"checkpoint_ts":457114753672675329,"checkpoint_time":"2025-04-04 16:13:20.912","gid":{"low":9242058342016720667,"high":3463320182788524165}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + set +x + ans='[ { "id": "065c48ff-2202-499a-8dc8-e25a7aa43b31", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "47106095-61c9-4e83-944a-90ea4be3361e", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"065c48ff-2202-499a-8dc8-e25a7aa43b31",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"47106095-61c9-4e83-944a-90ea4be3361e",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=47106095-61c9-4e83-944a-90ea4be3361e + '[' -z 47106095-61c9-4e83-944a-90ea4be3361e ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 47106095-61c9-4e83-944a-90ea4be3361e + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.cli.21089.out cli changefeed move-table -c test -t 114 -d 47106095-61c9-4e83-944a-90ea4be3361e command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' TEST FAILED: OUTPUT CONTAINS 'test1' ____________________________________ *************************** 1. row *************************** Tables_in_fail_over_ddl_test: test1 *************************** 2. row *************************** Tables_in_fail_over_ddl_test: test2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later + set +x Command succeeded on attempt 1 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table fail_over_ddl_test.test2 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully table fail_over_ddl_test.test2 exists + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (19408) - No such process wait process 19408 exit for 1-th time... process 19408 already exit cdc.test: no process found process cdc.test already exit [Fri Apr 4 16:13:29 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1954119543.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (19092) - No such process wait process 19092 exit for 1-th time... process 19092 already exit [Fri Apr 4 16:13:27 CST 2025] <<<<<< START cdc server in fail_over_ddl_L case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_L.1930419306.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_L/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_L/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:30 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3564 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5013aeab-9077-46fd-ab59-d77813973bd6 {"id":"5013aeab-9077-46fd-ab59-d77813973bd6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754377} /tidb/cdc/default/__cdc_meta__/capture/fb25c9c7-5166-4ade-a871-886e205db1c5 {"id":"fb25c9c7-5166-4ade-a871-886e205db1c5","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754408} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde1b85 fb25c9c7-5166-4ade-a871-886e205db1c5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde1b85 fb25c9c7-5166-4ade-a871-886e205db1c5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13423124298916295285,"high":16500626430214257564},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367931676216211,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:51.57902021+08:00","start-ts":457114744731205633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745976389640} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755007512580,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5013aeab-9077-46fd-ab59-d77813973bd6 {"id":"5013aeab-9077-46fd-ab59-d77813973bd6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754377} /tidb/cdc/default/__cdc_meta__/capture/fb25c9c7-5166-4ade-a871-886e205db1c5 {"id":"fb25c9c7-5166-4ade-a871-886e205db1c5","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754408} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde1b85 fb25c9c7-5166-4ade-a871-886e205db1c5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde1b85 fb25c9c7-5166-4ade-a871-886e205db1c5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13423124298916295285,"high":16500626430214257564},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367931676216211,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:51.57902021+08:00","start-ts":457114744731205633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745976389640} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755007512580,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5013aeab-9077-46fd-ab59-d77813973bd6 {"id":"5013aeab-9077-46fd-ab59-d77813973bd6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754377} /tidb/cdc/default/__cdc_meta__/capture/fb25c9c7-5166-4ade-a871-886e205db1c5 {"id":"fb25c9c7-5166-4ade-a871-886e205db1c5","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754408} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbde1b85 fb25c9c7-5166-4ade-a871-886e205db1c5 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde15ec 5013aeab-9077-46fd-ab59-d77813973bd6 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbde1b85 fb25c9c7-5166-4ade-a871-886e205db1c5 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":13423124298916295285,"high":16500626430214257564},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367931676216211,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:51.57902021+08:00","start-ts":457114744731205633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114745976389640} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755007512580,"progress":1}' + grep -q 'etcd info' + break + set +x /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (33325) - No such process wait process 33325 exit for 1-th time... process 33325 already exit /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (33408) - No such process wait process 33408 exit for 1-th time... process 33408 already exit [Fri Apr 4 16:13:28 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.3359933601.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3086 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e34fef26-e67a-4fde-b46a-c058a2bc7748 {"id":"e34fef26-e67a-4fde-b46a-c058a2bc7748","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754409} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114751203016714,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e34fef26-e67a-4fde-b46a-c058a2bc7748 {"id":"e34fef26-e67a-4fde-b46a-c058a2bc7748","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754409} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114751203016714,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e34fef26-e67a-4fde-b46a-c058a2bc7748 {"id":"e34fef26-e67a-4fde-b46a-c058a2bc7748","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754409} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114751203016714,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 1833 0 --:--:-- --:--:-- --:--:-- 1833 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:13:31 CST 2025] <<<<<< START cdc server in fail_over_ddl_M case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_M.3367133673.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_M/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_M/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3085 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2355bf2f-0834-465a-8c58-1c0e529f314e {"id":"2355bf2f-0834-465a-8c58-1c0e529f314e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754410} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9242058342016720667,"high":3463320182788524165},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368078723936793,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:25.652264049+08:00","start-ts":457114753672675329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114754904752138} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755271753748,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2355bf2f-0834-465a-8c58-1c0e529f314e {"id":"2355bf2f-0834-465a-8c58-1c0e529f314e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754410} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9242058342016720667,"high":3463320182788524165},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368078723936793,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:25.652264049+08:00","start-ts":457114753672675329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114754904752138} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755271753748,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2355bf2f-0834-465a-8c58-1c0e529f314e {"id":"2355bf2f-0834-465a-8c58-1c0e529f314e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754410} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9242058342016720667,"high":3463320182788524165},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368078723936793,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:25.652264049+08:00","start-ts":457114753672675329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114754904752138} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755271753748,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:13:32 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1962019622.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 command: run_sql 'use fail_over_ddl_test;show tables;' 127.0.0.1 3306 && check_not_contains 'test1' && check_contains 'test4' run task successfully check diff failed 1-th time, retry later command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (20916) - No such process wait process 20916 exit for 1-th time... process 20916 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 check diff failed 2-th time, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:34 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3667 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1aa1ba94-567e-4677-bfab-7aa61f4d3bb1 {"id":"1aa1ba94-567e-4677-bfab-7aa61f4d3bb1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754412} /tidb/cdc/default/__cdc_meta__/capture/e34fef26-e67a-4fde-b46a-c058a2bc7748 {"id":"e34fef26-e67a-4fde-b46a-c058a2bc7748","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754409} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c935 1aa1ba94-567e-4677-bfab-7aa61f4d3bb1 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c935 1aa1ba94-567e-4677-bfab-7aa61f4d3bb1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114751203016714,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1aa1ba94-567e-4677-bfab-7aa61f4d3bb1 {"id":"1aa1ba94-567e-4677-bfab-7aa61f4d3bb1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754412} /tidb/cdc/default/__cdc_meta__/capture/e34fef26-e67a-4fde-b46a-c058a2bc7748 {"id":"e34fef26-e67a-4fde-b46a-c058a2bc7748","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754409} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c935 1aa1ba94-567e-4677-bfab-7aa61f4d3bb1 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c935 1aa1ba94-567e-4677-bfab-7aa61f4d3bb1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114751203016714,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1aa1ba94-567e-4677-bfab-7aa61f4d3bb1 {"id":"1aa1ba94-567e-4677-bfab-7aa61f4d3bb1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754412} /tidb/cdc/default/__cdc_meta__/capture/e34fef26-e67a-4fde-b46a-c058a2bc7748 {"id":"e34fef26-e67a-4fde-b46a-c058a2bc7748","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754409} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c3c4 5fe1f0a4-edd7-4949-95ef-22aa18d0f77d /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdbf1c935 1aa1ba94-567e-4677-bfab-7aa61f4d3bb1 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c8d2 e34fef26-e67a-4fde-b46a-c058a2bc7748 /tidb/cdc/default/__cdc_meta__/owner/223195ffdbf1c935 1aa1ba94-567e-4677-bfab-7aa61f4d3bb1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":15438872866450531533,"high":2050275118990919340},"display":{"name":"test","namespace":"default"}},"upstream-id":7489367952783173265,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:12:56.100704032+08:00","start-ts":457114745920815105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114747165999109} /tidb/cdc/default/default/changefeed/status/test + grep -q 'etcd info' {"checkpoint-ts":457114751203016714,"progress":1}' + break + set +x command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully check diff failed 1-th time, retry later + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 91000 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2355bf2f-0834-465a-8c58-1c0e529f314e {"id":"2355bf2f-0834-465a-8c58-1c0e529f314e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754410} /tidb/cdc/default/__cdc_meta__/capture/8a0ea7f7-766c-4b37-84b7-554f51034684 {"id":"8a0ea7f7-766c-4b37-84b7-554f51034684","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754413} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9242058342016720667,"high":3463320182788524165},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368078723936793,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:25.652264049+08:00","start-ts":457114753672675329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114754904752138} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755297968129,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2355bf2f-0834-465a-8c58-1c0e529f314e {"id":"2355bf2f-0834-465a-8c58-1c0e529f314e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754410} /tidb/cdc/default/__cdc_meta__/capture/8a0ea7f7-766c-4b37-84b7-554f51034684 {"id":"8a0ea7f7-766c-4b37-84b7-554f51034684","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754413} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9242058342016720667,"high":3463320182788524165},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368078723936793,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:25.652264049+08:00","start-ts":457114753672675329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114754904752138} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755297968129,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2355bf2f-0834-465a-8c58-1c0e529f314e {"id":"2355bf2f-0834-465a-8c58-1c0e529f314e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754410} /tidb/cdc/default/__cdc_meta__/capture/8a0ea7f7-766c-4b37-84b7-554f51034684 {"id":"8a0ea7f7-766c-4b37-84b7-554f51034684","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754413} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673ad8 fe9647aa-9dbb-4a4e-a803-404c0cc46910 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673bb6 2355bf2f-0834-465a-8c58-1c0e529f314e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9242058342016720667,"high":3463320182788524165},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368078723936793,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:25.652264049+08:00","start-ts":457114753672675329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114754904752138} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755297968129,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.19694.out cli capture list check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseL-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_L/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_L/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_L/stdout1-1.log /tmp/tidb_cdc_test/fail_over_ddl_L/stdout1-2.log no DATA RACE found [Fri Apr 4 16:13:37 CST 2025] <<<<<< run test case fail_over_ddl_L success! >>>>>> cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseN-3 passed successfully shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x + ans='[ { "id": "2355bf2f-0834-465a-8c58-1c0e529f314e", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "8a0ea7f7-766c-4b37-84b7-554f51034684", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"2355bf2f-0834-465a-8c58-1c0e529f314e",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"8a0ea7f7-766c-4b37-84b7-554f51034684",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=8a0ea7f7-766c-4b37-84b7-554f51034684 + '[' -z 8a0ea7f7-766c-4b37-84b7-554f51034684 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 8a0ea7f7-766c-4b37-84b7-554f51034684 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.cli.19723.out cli changefeed move-table -c test -t 114 -d 8a0ea7f7-766c-4b37-84b7-554f51034684 check diff failed 2-th time, retry later + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:13:39 CST 2025] <<<<<< START cdc server in fail_over_ddl_B case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_B.2121021212.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_B/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_B/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseM-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_M/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_M/stdout0-2.log /tmp/tidb_cdc_test/fail_over_ddl_M/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_M/stdout1-1.log /tmp/tidb_cdc_test/fail_over_ddl_M/stdout1-2.log no DATA RACE found [Fri Apr 4 16:13:39 CST 2025] <<<<<< run test case fail_over_ddl_M success! >>>>>> + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:42 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3666 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47106095-61c9-4e83-944a-90ea4be3361e {"id":"47106095-61c9-4e83-944a-90ea4be3361e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754401} /tidb/cdc/default/__cdc_meta__/capture/a54d70f5-8f35-408d-b5f1-85804ac7784d {"id":"a54d70f5-8f35-408d-b5f1-85804ac7784d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754419} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc351918 a54d70f5-8f35-408d-b5f1-85804ac7784d /tidb/cdc/default/__cdc_meta__/owner/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc351918 a54d70f5-8f35-408d-b5f1-85804ac7784d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8088882069560617283,"high":5566475742481669810},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368032724137380,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:14.260802664+08:00","start-ts":457114750688165889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751920504842} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755983474701,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47106095-61c9-4e83-944a-90ea4be3361e {"id":"47106095-61c9-4e83-944a-90ea4be3361e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754401} /tidb/cdc/default/__cdc_meta__/capture/a54d70f5-8f35-408d-b5f1-85804ac7784d {"id":"a54d70f5-8f35-408d-b5f1-85804ac7784d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754419} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc351918 a54d70f5-8f35-408d-b5f1-85804ac7784d /tidb/cdc/default/__cdc_meta__/owner/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc351918 a54d70f5-8f35-408d-b5f1-85804ac7784d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8088882069560617283,"high":5566475742481669810},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368032724137380,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:14.260802664+08:00","start-ts":457114750688165889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751920504842} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755983474701,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47106095-61c9-4e83-944a-90ea4be3361e {"id":"47106095-61c9-4e83-944a-90ea4be3361e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754401} /tidb/cdc/default/__cdc_meta__/capture/a54d70f5-8f35-408d-b5f1-85804ac7784d {"id":"a54d70f5-8f35-408d-b5f1-85804ac7784d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754419} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc3515c0 065c48ff-2202-499a-8dc8-e25a7aa43b31 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc351918 a54d70f5-8f35-408d-b5f1-85804ac7784d /tidb/cdc/default/__cdc_meta__/owner/223195ffdc351624 47106095-61c9-4e83-944a-90ea4be3361e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc351918 a54d70f5-8f35-408d-b5f1-85804ac7784d /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":8088882069560617283,"high":5566475742481669810},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368032724137380,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:14.260802664+08:00","start-ts":457114750688165889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751920504842} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755983474701,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully check diff failed 2-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/fail_over_ddl_N Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18986) - No such process wait process 18986 exit for 1-th time... process 18986 already exit [Fri Apr 4 16:13:44 CST 2025] <<<<<< START cdc server in fail_over_ddl_O case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_O.1917419176.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_O/cdc1-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_O/cdc_data1-2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:13:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3565 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22fdf121-935b-42bc-889f-6ce21fc3d409 {"id":"22fdf121-935b-42bc-889f-6ce21fc3d409","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754396} /tidb/cdc/default/__cdc_meta__/capture/483c2cd2-3581-4e48-b79c-039a930b822e {"id":"483c2cd2-3581-4e48-b79c-039a930b822e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754424} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c6d2 483c2cd2-3581-4e48-b79c-039a930b822e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c6d2 483c2cd2-3581-4e48-b79c-039a930b822e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16087946732044607413,"high":17203305829823890572},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368023286308762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:11.918771711+08:00","start-ts":457114750070554625,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751316000770} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755366125574,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22fdf121-935b-42bc-889f-6ce21fc3d409 {"id":"22fdf121-935b-42bc-889f-6ce21fc3d409","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754396} /tidb/cdc/default/__cdc_meta__/capture/483c2cd2-3581-4e48-b79c-039a930b822e {"id":"483c2cd2-3581-4e48-b79c-039a930b822e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754424} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c6d2 483c2cd2-3581-4e48-b79c-039a930b822e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c6d2 483c2cd2-3581-4e48-b79c-039a930b822e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16087946732044607413,"high":17203305829823890572},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368023286308762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:11.918771711+08:00","start-ts":457114750070554625,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751316000770} /tidb/cdc/default/default/changefeed/status/test + grep -q 'failed to get info:' {"checkpoint-ts":457114755366125574,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22fdf121-935b-42bc-889f-6ce21fc3d409 {"id":"22fdf121-935b-42bc-889f-6ce21fc3d409","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754396} /tidb/cdc/default/__cdc_meta__/capture/483c2cd2-3581-4e48-b79c-039a930b822e {"id":"483c2cd2-3581-4e48-b79c-039a930b822e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754424} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc31c6d2 483c2cd2-3581-4e48-b79c-039a930b822e /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c1c1 22fdf121-935b-42bc-889f-6ce21fc3d409 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc31c6d2 483c2cd2-3581-4e48-b79c-039a930b822e /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":16087946732044607413,"high":17203305829823890572},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368023286308762,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:11.918771711+08:00","start-ts":457114750070554625,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114751316000770} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114755366125574,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseB-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_B/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_B/stdout0-2.log /tmp/tidb_cdc_test/fail_over_ddl_B/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_B/stdout1-1.log no DATA RACE found [Fri Apr 4 16:13:47 CST 2025] <<<<<< run test case fail_over_ddl_B success! >>>>>> [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368243802039643 TiDB Cluster ID. tikv_gc_leader_uuid 657ff7465d00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:17007, start at 2025-04-04 16:13:54.963795192 +0800 CST m=+3.003059429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:55.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:55.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:55.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368243802039643 TiDB Cluster ID. tikv_gc_leader_uuid 657ff7465d00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:17007, start at 2025-04-04 16:13:54.963795192 +0800 CST m=+3.003059429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:55.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:55.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:55.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489368243802685262 TiDB Cluster ID. tikv_gc_leader_uuid 657ff746a6c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-ticdc-pull-cdc-mysql-integration-light-358-n1qgb--vb16s, pid:17067, start at 2025-04-04 16:13:55.239256101 +0800 CST m=+3.173077399 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-16:15:55.246 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-16:13:55.227 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-16:03:55.227 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/fail_over_ddl_N/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (19546) - No such process wait process 19546 exit for 1-th time... process 19546 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } check diff successfully + ipAddr=127.0.0.1:8300 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } process cdc.test already exit failOverCaseO-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_O/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_O/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_O/stdout1-1.log /tmp/tidb_cdc_test/fail_over_ddl_O/stdout1-2.log no DATA RACE found [Fri Apr 4 16:13:58 CST 2025] <<<<<< run test case fail_over_ddl_O success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 ++ grep -v 'Command to ticdc' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.18290.out cli tso query --pd=http://127.0.0.1:2379 + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + set +x + tso='457114763626283009 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/...' + echo 457114763626283009 PASS coverage: 2.0% of statements in github.com/pingcap/ticdc/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 16:14:00 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1832518327.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:14:01 CST 2025] <<<<<< START cdc server in fail_over_ddl_I case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforePass=sleep(60000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_I.1984719849.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_I/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_I/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:14:03 GMT < Content-Length: 603 < Content-Type: text/plain; charset=utf-8 < { [603 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 {"id":"42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754440} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 {"id":"42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754440} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 {"id":"42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754440} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0' + grep -q 'etcd info' + break + set +x Attempt 1 of 5: run_cdc_cli changefeed create --start-ts=457114763626283009 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.18383.out cli changefeed create --start-ts=457114763626283009 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489368243802039643,"id":"test","namespace":"default","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T16:14:03.630257572+08:00","start_ts":457114763626283009,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0,"split_number_per_node":1},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-55-g52a1009d","resolved_ts":457114763626283009,"checkpoint_ts":457114763626283009,"checkpoint_time":"2025-04-04 16:13:58.882","gid":{"low":14143699623522126984,"high":17380438190354515645}} PASS coverage: 2.7% of statements in github.com/pingcap/ticdc/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:14:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3563 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a0ea7f7-766c-4b37-84b7-554f51034684 {"id":"8a0ea7f7-766c-4b37-84b7-554f51034684","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754413} /tidb/cdc/default/__cdc_meta__/capture/a8550cec-8741-4203-a8fe-f4d6c5a2a560 {"id":"a8550cec-8741-4203-a8fe-f4d6c5a2a560","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754441} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc6740f1 a8550cec-8741-4203-a8fe-f4d6c5a2a560 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc6740f1 a8550cec-8741-4203-a8fe-f4d6c5a2a560 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9242058342016720667,"high":3463320182788524165},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368078723936793,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:25.652264049+08:00","start-ts":457114753672675329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114754904752138} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114762100342785,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a0ea7f7-766c-4b37-84b7-554f51034684 {"id":"8a0ea7f7-766c-4b37-84b7-554f51034684","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754413} /tidb/cdc/default/__cdc_meta__/capture/a8550cec-8741-4203-a8fe-f4d6c5a2a560 {"id":"a8550cec-8741-4203-a8fe-f4d6c5a2a560","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754441} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc6740f1 a8550cec-8741-4203-a8fe-f4d6c5a2a560 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc6740f1 a8550cec-8741-4203-a8fe-f4d6c5a2a560 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9242058342016720667,"high":3463320182788524165},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368078723936793,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:25.652264049+08:00","start-ts":457114753672675329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114754904752138} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114762100342785,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a0ea7f7-766c-4b37-84b7-554f51034684 {"id":"8a0ea7f7-766c-4b37-84b7-554f51034684","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754413} /tidb/cdc/default/__cdc_meta__/capture/a8550cec-8741-4203-a8fe-f4d6c5a2a560 {"id":"a8550cec-8741-4203-a8fe-f4d6c5a2a560","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754441} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdc6740f1 a8550cec-8741-4203-a8fe-f4d6c5a2a560 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc673c1e 8a0ea7f7-766c-4b37-84b7-554f51034684 /tidb/cdc/default/__cdc_meta__/owner/223195ffdc6740f1 a8550cec-8741-4203-a8fe-f4d6c5a2a560 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":9242058342016720667,"high":3463320182788524165},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368078723936793,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:13:25.652264049+08:00","start-ts":457114753672675329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114754904752138} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114762100342785,"progress":1}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later + set +x Command succeeded on attempt 1 table fail_over_ddl_test.test2 not exists for 1-th check, retry later check diff failed 2-th time, retry later table fail_over_ddl_test.test2 exists /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18330) - No such process wait process 18330 exit for 1-th time... process 18330 already exit [Fri Apr 4 16:14:07 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1847518477.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0-1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 [Pipeline] } [Pipeline] // stage [Pipeline] } cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseI-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_I/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_I/stdout0-2.log /tmp/tidb_cdc_test/fail_over_ddl_I/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_I/stdout1-1.log no DATA RACE found [Fri Apr 4 16:14:09 CST 2025] <<<<<< run test case fail_over_ddl_I success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:14:10 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3087 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea80829a-73ee-4a6a-9a01-06d9d95b7738 {"id":"ea80829a-73ee-4a6a-9a01-06d9d95b7738","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754448} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":14143699623522126984,"high":17380438190354515645},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368243802039643,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:14:03.630257572+08:00","start-ts":457114763626283009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114764871467010} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114765238468643,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea80829a-73ee-4a6a-9a01-06d9d95b7738 {"id":"ea80829a-73ee-4a6a-9a01-06d9d95b7738","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754448} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":14143699623522126984,"high":17380438190354515645},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368243802039643,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:14:03.630257572+08:00","start-ts":457114763626283009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114764871467010} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114765238468643,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea80829a-73ee-4a6a-9a01-06d9d95b7738 {"id":"ea80829a-73ee-4a6a-9a01-06d9d95b7738","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754448} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":14143699623522126984,"high":17380438190354515645},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368243802039643,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:14:03.630257572+08:00","start-ts":457114763626283009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114764871467010} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114765238468643,"progress":1}' + grep -q 'etcd info' + break + set +x + ipAddr=127.0.0.1:8300 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8300 test -1 60 + ipAddr=127.0.0.1:8300 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 2200 0 --:--:-- --:--:-- --:--:-- 2200 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:14:10 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/ticdc/pkg/scheduler/StopBalanceScheduler=return(true);github.com/pingcap/ticdc/downstreamadapter/dispatcher/BlockOrWaitBeforeDealWithDDL=1*sleep(30000)->return(false)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1855818560.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc1-1.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data1-1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:14:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3668 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 {"id":"214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754451} /tidb/cdc/default/__cdc_meta__/capture/ea80829a-73ee-4a6a-9a01-06d9d95b7738 {"id":"ea80829a-73ee-4a6a-9a01-06d9d95b7738","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754448} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":14143699623522126984,"high":17380438190354515645},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368243802039643,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:14:03.630257572+08:00","start-ts":457114763626283009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114764871467010} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114767073476623,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 {"id":"214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754451} /tidb/cdc/default/__cdc_meta__/capture/ea80829a-73ee-4a6a-9a01-06d9d95b7738 {"id":"ea80829a-73ee-4a6a-9a01-06d9d95b7738","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754448} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":14143699623522126984,"high":17380438190354515645},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368243802039643,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:14:03.630257572+08:00","start-ts":457114763626283009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114764871467010} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114767073476623,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 {"id":"214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754451} /tidb/cdc/default/__cdc_meta__/capture/ea80829a-73ee-4a6a-9a01-06d9d95b7738 {"id":"ea80829a-73ee-4a6a-9a01-06d9d95b7738","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754448} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e1e5 42cd7ea4-fdf7-41ac-8854-aa1d56fc6fa0 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e2c4 ea80829a-73ee-4a6a-9a01-06d9d95b7738 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":14143699623522126984,"high":17380438190354515645},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368243802039643,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:14:03.630257572+08:00","start-ts":457114763626283009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114764871467010} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114767073476623,"progress":1}' + grep -q 'etcd info' + break + set +x + dbName=fail_over_ddl_test + tableName=test1 + MAX_RETRIES=5 + retries=0 + '[' 0 -lt 5 ']' ++ curl http://127.0.0.1:10080/schema/fail_over_ddl_test/test1 ++ jq .id % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2126 0 2126 0 0 2076k 0 --:--:-- --:--:-- --:--:-- 2076k + id=114 + '[' -n 114 ']' + echo 114 + exit 0 + ipAddr=127.0.0.1:8301 + tableID=114 + changefeedID=test + retryCount=10 + echo 'move table with retry' move table with retry + count=0 + [[ 0 -lt 10 ]] ++ run_cdc_cli capture list + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.18635.out cli capture list + set +x + ans='[ { "id": "ea80829a-73ee-4a6a-9a01-06d9d95b7738", "is_owner": true, "address": "127.0.0.1:8300", "cluster_id": "default" }, { "id": "214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1", "is_owner": false, "address": "127.0.0.1:8301", "cluster_id": "default" } ] PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' ++ echo '[' '{' '"id":' '"ea80829a-73ee-4a6a-9a01-06d9d95b7738",' '"is_owner":' true, '"address":' '"127.0.0.1:8300",' '"cluster_id":' '"default"' '},' '{' '"id":' '"214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1",' '"is_owner":' false, '"address":' '"127.0.0.1:8301",' '"cluster_id":' '"default"' '}' ']' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r '.[] | select(.address == "127.0.0.1:8301") | .id' + node2ID=214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 + '[' -z 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 ']' ++ run_cdc_cli changefeed move-table -c test -t 114 -d 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.cli.18666.out cli changefeed move-table -c test -t 114 -d 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 + set +x + result='{ "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/...' + echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... { "success": true, "error": "" } PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ echo '{' '"success":' true, '"error":' '""' '}' PASS coverage: 2.2% of statements in github.com/pingcap/ticdc/... ++ sed 's/ PASS.*//' ++ sed 's/^=== Command to ticdc(new arch). //' ++ jq -r .success + success=true + '[' true == true ']' + exit 0 command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' TEST FAILED: OUTPUT DOES NOT CONTAIN '2' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_contains '2' run task successfully /home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (18480) - No such process wait process 18480 exit for 1-th time... process 18480 already exit + ipAddr=127.0.0.1:8301 + changefeedID=test + retryCount=60 + echo 'check coordinator and maintainer' check coordinator and maintainer + query_dispatcher_count 127.0.0.1:8301 test -1 60 + ipAddr=127.0.0.1:8301 + changefeedID=test + target=-1 + retryCount=60 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 22750 0 --:--:-- --:--:-- --:--:-- 22750 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=1 + [[ 1 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 91000 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=2 + [[ 2 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 91 100 91 0 0 45500 0 --:--:-- --:--:-- --:--:-- 45500 + ans='{ "error_msg": "Can'\''t not find maintainer for changefeed: test", "error_code": "" }' + echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' { "error_msg": "Can't not find maintainer for changefeed: test", "error_code": "" } ++ echo '{' '"error_msg":' '"Can'\''t' not find maintainer for changefeed: 'test",' '"error_code":' '""' '}' ++ jq -r .count + value=null + '[' -1 == -1 ']' + '[' null '!=' null ']' + sleep 2 + count=3 + [[ 3 -lt 60 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 3666 + ans='{"count":3}' + echo '{"count":3}' {"count":3} ++ echo '{"count":3}' ++ jq -r .count + value=3 + '[' -1 == -1 ']' + '[' 3 '!=' null ']' + exit 0 [Fri Apr 4 16:14:41 CST 2025] <<<<<< START cdc server in fail_over_ddl_N case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.fail_over_ddl_N.1881318815.out server --log-file /tmp/tidb_cdc_test/fail_over_ddl_N/cdc0-2.log --log-level debug --data-dir /tmp/tidb_cdc_test/fail_over_ddl_N/cdc_data0-2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 08:14:44 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3565 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 {"id":"214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754451} /tidb/cdc/default/__cdc_meta__/capture/d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf {"id":"d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754482} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e87e d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e87e d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":14143699623522126984,"high":17380438190354515645},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368243802039643,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:14:03.630257572+08:00","start-ts":457114763626283009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114764871467010} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114769170628628,"progress":1}' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 {"id":"214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754451} /tidb/cdc/default/__cdc_meta__/capture/d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf {"id":"d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754482} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e87e d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e87e d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":14143699623522126984,"high":17380438190354515645},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368243802039643,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:14:03.630257572+08:00","start-ts":457114763626283009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114764871467010} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114769170628628,"progress":1}' + grep -q 'failed to get info:' + echo ' *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 {"id":"214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754451} /tidb/cdc/default/__cdc_meta__/capture/d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf {"id":"d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-55-g52a1009d","git-hash":"52a1009d7bcd84b283c5443601c9b3160b35bb7c","deploy-path":"/home/jenkins/agent/workspace/pingcap/ticdc/pull_cdc_mysql_integration_light/ticdc/bin/cdc.test","start-timestamp":1743754482} /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/__cdc_meta__/log_coordinator/223195ffdcf5e87e d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e328 214e4ad3-56d5-4b0f-8511-d2a08d4e4fa1 /tidb/cdc/default/__cdc_meta__/owner/223195ffdcf5e87e d23f6dc1-45a7-4edc-9fbb-e68ae9dd8bdf /tidb/cdc/default/default/changefeed/info/test {"id":{"id":{"low":14143699623522126984,"high":17380438190354515645},"display":{"name":"test","namespace":"default"}},"upstream-id":7489368243802039643,"sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T16:14:03.630257572+08:00","start-ts":457114763626283009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","upstream-info":null,"config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true},"case-sensitive":false,"integrity":null,"force-replicate":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"split-number-per-node":1},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-55-g52a1009d","epoch":457114764871467010} /tidb/cdc/default/default/changefeed/status/test {"checkpoint-ts":457114769170628628,"progress":1}' + grep -q 'etcd info' + break + set +x command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' TEST FAILED: OUTPUT CONTAINS '2' ____________________________________ *************************** 1. row *************************** id: 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later command: run_sql 'select id from fail_over_ddl_test.test1;' 127.0.0.1 3306 && check_not_contains '2' run task successfully check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully + ipAddr=127.0.0.1:8301 + changefeedID=test + target=5 + retryCount=10 + echo 'query dispatcher count' query dispatcher count + count=0 + [[ 0 -lt 10 ]] ++ curl -X GET http://127.0.0.1:8301/api/v2/changefeeds/test/get_dispatcher_count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 11 100 11 0 0 3666 0 --:--:-- --:--:-- --:--:-- 5500 + ans='{"count":5}' + echo '{"count":5}' {"count":5} ++ echo '{"count":5}' ++ jq -r .count + value=5 + '[' 5 == -1 ']' + '[' 5 == 5 ']' + exit 0 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit failOverCaseN-4 passed successfully log files: /tmp/tidb_cdc_test/fail_over_ddl_N/stdout0-1.log /tmp/tidb_cdc_test/fail_over_ddl_N/stdout0-2.log /tmp/tidb_cdc_test/fail_over_ddl_N/stdout0.log /tmp/tidb_cdc_test/fail_over_ddl_N/stdout1-1.log no DATA RACE found [Fri Apr 4 16:15:02 CST 2025] <<<<<< run test case fail_over_ddl_N success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-ticdc-pull_cdc_mysql_integration_light-358/ticdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS